feat(report): Include dependencies into scan result and cyclondex for supply chain security on Integration with GitHub Security Alerts (#1584)

* feat(report): Enhance scan result and cyclondex for supply chain security on Integration with GitHub Security Alerts

* derive ecosystem/version from dependency graph

* fix vars name && fetch manifest info on GSA && arrange ghpkgToPURL structure

* fix miscs

* typo in error message

* fix ecosystem equally to trivy

* miscs

* refactoring

* recursive dependency graph pagination

* change var name && update comments

* omit map type of ghpkgToPURL in signatures

* fix vars name

* goimports

* make fmt

* fix comment

Co-authored-by: MaineK00n <[email protected]>

Author: kl-sinclair

cache/bolt.go

@@ -48,7 +48,7 @@ func (b Bolt) Close() error {
 	return b.db.Close()
 }
 
-//  CreateBucketIfNotExists creates a bucket that is specified by arg.
+// CreateBucketIfNotExists creates a bucket that is specified by arg.
 func (b *Bolt) createBucketIfNotExists(name string) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(name))

config/config.go

@@ -21,7 +21,7 @@ var Revision string
 // Conf has Configuration
 var Conf Config
 
-//Config is struct of Configuration
+// Config is struct of Configuration
 type Config struct {
 	logging.LogOpts
 

detector/detector.go

@@ -303,6 +303,10 @@ func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]config.GitHub
 		}
 		logging.Log.Infof("%s: %d CVEs detected with GHSA %s/%s",
 			r.FormatServerName(), n, owner, repo)
+
+		if err = DetectGitHubDependencyGraph(r, owner, repo, setting.Token); err != nil {
+			return xerrors.Errorf("Failed to access GitHub Dependency graph: %w", err)
+		}
 	}
 	return nil
 }

detector/github.go

@@ -29,7 +29,7 @@ func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string,
 	// TODO Use `https://github.com/shurcooL/githubv4` if the tool supports vulnerabilityAlerts Endpoint
 	// Memo : https://developer.github.com/v4/explorer/
 	const jsonfmt = `{"query":
-	"query { repository(owner:\"%s\", name:\"%s\") { url vulnerabilityAlerts(first: %d, states:[OPEN], %s) { pageInfo { endCursor hasNextPage startCursor } edges { node { id dismissReason dismissedAt securityVulnerability{ package { name ecosystem } severity vulnerableVersionRange firstPatchedVersion { identifier } } securityAdvisory { description ghsaId permalink publishedAt summary updatedAt withdrawnAt origin severity references { url } identifiers { type value } } } } } } } "}`
+	"query { repository(owner:\"%s\", name:\"%s\") { url vulnerabilityAlerts(first: %d, states:[OPEN], %s) { pageInfo { endCursor hasNextPage startCursor } edges { node { id dismissReason dismissedAt securityVulnerability{ package { name ecosystem } severity vulnerableVersionRange firstPatchedVersion { identifier } } vulnerableManifestFilename vulnerableManifestPath vulnerableRequirements securityAdvisory { description ghsaId permalink publishedAt summary updatedAt withdrawnAt origin severity references { url } identifiers { type value } } } } } } } "}`
 	after := ""
 
 	for {
@@ -79,11 +79,19 @@ func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string,
 				continue
 			}
 
-			pkgName := fmt.Sprintf("%s %s",
+			repoURLPkgName := fmt.Sprintf("%s %s",
 				alerts.Data.Repository.URL, v.Node.SecurityVulnerability.Package.Name)
 
 			m := models.GitHubSecurityAlert{
-				PackageName:   pkgName,
+				PackageName: repoURLPkgName,
+				Repository:  alerts.Data.Repository.URL,
+				Package: models.GSAVulnerablePackage{
+					Name:             v.Node.SecurityVulnerability.Package.Name,
+					Ecosystem:        v.Node.SecurityVulnerability.Package.Ecosystem,
+					ManifestFilename: v.Node.VulnerableManifestFilename,
+					ManifestPath:     v.Node.VulnerableManifestPath,
+					Requirements:     v.Node.VulnerableRequirements,
+				},
 				FixedIn:       v.Node.SecurityVulnerability.FirstPatchedVersion.Identifier,
 				AffectedRange: v.Node.SecurityVulnerability.VulnerableVersionRange,
 				Dismissed:     len(v.Node.DismissReason) != 0,
@@ -175,7 +183,10 @@ type SecurityAlerts struct {
 								Identifier string `json:"identifier"`
 							} `json:"firstPatchedVersion"`
 						} `json:"securityVulnerability"`
-						SecurityAdvisory struct {
+						VulnerableManifestFilename string `json:"vulnerableManifestFilename"`
+						VulnerableManifestPath     string `json:"vulnerableManifestPath"`
+						VulnerableRequirements     string `json:"vulnerableRequirements"`
+						SecurityAdvisory           struct {
 							Description string    `json:"description"`
 							GhsaID      string    `json:"ghsaId"`
 							Permalink   string    `json:"permalink"`
@@ -199,3 +210,137 @@ type SecurityAlerts struct {
 		} `json:"repository"`
 	} `json:"data"`
 }
+
+// DetectGitHubDependencyGraph access to owner/repo on GitHub and fetch dependency graph of the repository via GitHub API v4 GraphQL and then set to the given ScanResult.
+// https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-the-dependency-graph
+func DetectGitHubDependencyGraph(r *models.ScanResult, owner, repo, token string) (err error) {
+	src := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: token},
+	)
+	//TODO Proxy
+	httpClient := oauth2.NewClient(context.Background(), src)
+	r.GitHubManifests = models.DependencyGraphManifests{}
+
+	return fetchDependencyGraph(r, httpClient, owner, repo, "", "")
+}
+
+// recursive function
+func fetchDependencyGraph(r *models.ScanResult, httpClient *http.Client, owner, repo, after, dependenciesAfter string) (err error) {
+	const queryFmt = `{"query":
+	"query { repository(owner:\"%s\", name:\"%s\") { url dependencyGraphManifests(first: %d, withDependencies: true%s) { pageInfo { endCursor hasNextPage } edges { node { blobPath filename repository { url } parseable exceedsMaxSize dependenciesCount dependencies%s { pageInfo { endCursor hasNextPage } edges { node { packageName packageManager repository { url } requirements hasDependencies } } } } } } } }"}`
+
+	queryStr := fmt.Sprintf(queryFmt, owner, repo, 100, after, dependenciesAfter)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		"https://api.github.com/graphql",
+		bytes.NewBuffer([]byte(queryStr)),
+	)
+	defer cancel()
+	if err != nil {
+		return err
+	}
+
+	// https://docs.github.com/en/graphql/overview/schema-previews#access-to-a-repository-s-dependency-graph-preview
+	// TODO remove this header if it is no longer preview status in the future.
+	req.Header.Set("Accept", "application/vnd.github.hawkgirl-preview+json")
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	graph := DependencyGraph{}
+	if err := json.Unmarshal(body, &graph); err != nil {
+		return err
+	}
+
+	if graph.Data.Repository.URL == "" {
+		return errof.New(errof.ErrFailedToAccessGithubAPI,
+			fmt.Sprintf("Failed to access to GitHub API. Response: %s", string(body)))
+	}
+
+	dependenciesAfter = ""
+	for _, m := range graph.Data.Repository.DependencyGraphManifests.Edges {
+		manifest, ok := r.GitHubManifests[m.Node.Filename]
+		if !ok {
+			manifest = models.DependencyGraphManifest{
+				Filename:     m.Node.Filename,
+				Repository:   m.Node.Repository.URL,
+				Dependencies: []models.Dependency{},
+			}
+		}
+		for _, d := range m.Node.Dependencies.Edges {
+			manifest.Dependencies = append(manifest.Dependencies, models.Dependency{
+				PackageName:    d.Node.PackageName,
+				PackageManager: d.Node.PackageManager,
+				Repository:     d.Node.Repository.URL,
+				Requirements:   d.Node.Requirements,
+			})
+		}
+		r.GitHubManifests[m.Node.Filename] = manifest
+
+		if m.Node.Dependencies.PageInfo.HasNextPage {
+			dependenciesAfter = fmt.Sprintf(`(after: \"%s\")`, m.Node.Dependencies.PageInfo.EndCursor)
+		}
+	}
+	if dependenciesAfter != "" {
+		return fetchDependencyGraph(r, httpClient, owner, repo, after, dependenciesAfter)
+	}
+
+	if graph.Data.Repository.DependencyGraphManifests.PageInfo.HasNextPage {
+		after = fmt.Sprintf(`, after: \"%s\"`, graph.Data.Repository.DependencyGraphManifests.PageInfo.EndCursor)
+		return fetchDependencyGraph(r, httpClient, owner, repo, after, dependenciesAfter)
+	}
+
+	return nil
+}
+
+type DependencyGraph struct {
+	Data struct {
+		Repository struct {
+			URL                      string `json:"url"`
+			DependencyGraphManifests struct {
+				PageInfo struct {
+					EndCursor   string `json:"endCursor"`
+					HasNextPage bool   `json:"hasNextPage"`
+				} `json:"pageInfo"`
+				Edges []struct {
+					Node struct {
+						BlobPath   string `json:"blobPath"`
+						Filename   string `json:"filename"`
+						Repository struct {
+							URL string `json:"url"`
+						}
+						Parseable         bool `json:"parseable"`
+						ExceedsMaxSize    bool `json:"exceedsMaxSize"`
+						DependenciesCount int  `json:"dependenciesCount"`
+						Dependencies      struct {
+							PageInfo struct {
+								EndCursor   string `json:"endCursor"`
+								HasNextPage bool   `json:"hasNextPage"`
+							} `json:"pageInfo"`
+							Edges []struct {
+								Node struct {
+									PackageName    string `json:"packageName"`
+									PackageManager string `json:"packageManager"`
+									Repository     struct {
+										URL string `json:"url"`
+									}
+									Requirements    string `json:"requirements"`
+									HasDependencies bool   `json:"hasDependencies"`
+								} `json:"node"`
+							} `json:"edges"`
+						} `json:"dependencies"`
+					} `json:"node"`
+				} `json:"edges"`
+			} `json:"dependencyGraphManifests"`
+		} `json:"repository"`
+	} `json:"data"`
+}

detector/wordpress.go

@@ -21,7 +21,7 @@ import (
 	"golang.org/x/xerrors"
 )
 
-//WpCveInfos is for wpscan json
+// WpCveInfos is for wpscan json
 type WpCveInfos struct {
 	ReleaseDate  string `json:"release_date"`
 	ChangelogURL string `json:"changelog_url"`
@@ -33,7 +33,7 @@ type WpCveInfos struct {
 	Error           string      `json:"error"`
 }
 
-//WpCveInfo is for wpscan json
+// WpCveInfo is for wpscan json
 type WpCveInfo struct {
 	ID         string     `json:"id"`
 	Title      string     `json:"title"`
@@ -44,7 +44,7 @@ type WpCveInfo struct {
 	FixedIn    string     `json:"fixed_in"`
 }
 
-//References is for wpscan json
+// References is for wpscan json
 type References struct {
 	URL     []string `json:"url"`
 	Cve     []string `json:"cve"`

logging/logutil.go

@@ -15,7 +15,7 @@ import (
 	formatter "github.com/kotakanbe/logrus-prefixed-formatter"
 )
 
-//LogOpts has options for logging
+// LogOpts has options for logging
 type LogOpts struct {
 	Debug     bool   `json:"debug,omitempty"`
 	DebugSQL  bool   `json:"debugSQL,omitempty"`
@@ -45,7 +45,6 @@ func NewNormalLogger() Logger {
 	return Logger{Entry: logrus.Entry{Logger: logrus.New()}}
 }
 
-// NewNormalLogger creates normal logger
 func NewIODiscardLogger() Logger {
 	l := logrus.New()
 	l.Out = io.Discard

models/github.go

@@ -0,0 +1,86 @@
+package models
+
+import (
+	"strings"
+
+	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
+)
+
+// DependencyGraphManifests has a map of DependencyGraphManifest
+// key: Filename
+type DependencyGraphManifests map[string]DependencyGraphManifest
+
+type DependencyGraphManifest struct {
+	Filename     string       `json:"filename"`
+	Repository   string       `json:"repository"`
+	Dependencies []Dependency `json:"dependencies"`
+}
+
+// Ecosystem returns a name of ecosystem(or package manager) of manifest(lock) file in trivy way
+// https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-the-dependency-graph#supported-package-ecosystems
+func (m DependencyGraphManifest) Ecosystem() string {
+	switch {
+	case strings.HasSuffix(m.Filename, "Cargo.lock"),
+		strings.HasSuffix(m.Filename, "Cargo.toml"):
+		return ftypes.Cargo // Rust
+	case strings.HasSuffix(m.Filename, "composer.lock"),
+		strings.HasSuffix(m.Filename, "composer.json"):
+		return ftypes.Composer // PHP
+	case strings.HasSuffix(m.Filename, ".csproj"),
+		strings.HasSuffix(m.Filename, ".vbproj"),
+		strings.HasSuffix(m.Filename, ".nuspec"),
+		strings.HasSuffix(m.Filename, ".vcxproj"),
+		strings.HasSuffix(m.Filename, ".fsproj"),
+		strings.HasSuffix(m.Filename, "packages.config"):
+		return ftypes.NuGet // .NET languages (C#, F#, VB), C++
+	case strings.HasSuffix(m.Filename, "go.sum"),
+		strings.HasSuffix(m.Filename, "go.mod"):
+		return ftypes.GoModule // Go
+	case strings.HasSuffix(m.Filename, "pom.xml"):
+		return ftypes.Pom // Java, Scala
+	case strings.HasSuffix(m.Filename, "package-lock.json"),
+		strings.HasSuffix(m.Filename, "package.json"):
+		return ftypes.Npm // JavaScript
+	case strings.HasSuffix(m.Filename, "yarn.lock"):
+		return ftypes.Yarn // JavaScript
+	case strings.HasSuffix(m.Filename, "requirements.txt"),
+		strings.HasSuffix(m.Filename, "requirements-dev.txt"),
+		strings.HasSuffix(m.Filename, "setup.py"):
+		return ftypes.Pip // Python
+	case strings.HasSuffix(m.Filename, "Pipfile.lock"),
+		strings.HasSuffix(m.Filename, "Pipfile"):
+		return ftypes.Pipenv // Python
+	case strings.HasSuffix(m.Filename, "poetry.lock"),
+		strings.HasSuffix(m.Filename, "pyproject.toml"):
+		return ftypes.Poetry // Python
+	case strings.HasSuffix(m.Filename, "Gemfile.lock"),
+		strings.HasSuffix(m.Filename, "Gemfile"):
+		return ftypes.Bundler // Ruby
+	case strings.HasSuffix(m.Filename, ".gemspec"):
+		return ftypes.GemSpec // Ruby
+	case strings.HasSuffix(m.Filename, "pubspec.lock"),
+		strings.HasSuffix(m.Filename, "pubspec.yaml"):
+		return "pub" // Dart
+	case strings.HasSuffix(m.Filename, ".yml"),
+		strings.HasSuffix(m.Filename, ".yaml"):
+		return "actions" // GitHub Actions workflows
+	default:
+		return "unknown"
+	}
+}
+
+type Dependency struct {
+	PackageName    string `json:"packageName"`
+	PackageManager string `json:"packageManager"`
+	Repository     string `json:"repository"`
+	Requirements   string `json:"requirements"`
+}
+
+func (d Dependency) Version() string {
+	s := strings.Split(d.Requirements, " ")
+	if len(s) == 2 && s[0] == "=" {
+		return s[1]
+	}
+	// in case of ranged version
+	return ""
+}

models/scanresults.go

@@ -45,15 +45,16 @@ type ScanResult struct {
 	Errors           []string          `json:"errors"`
 	Warnings         []string          `json:"warnings"`
 
-	ScannedCves       VulnInfos              `json:"scannedCves"`
-	RunningKernel     Kernel                 `json:"runningKernel"`
-	Packages          Packages               `json:"packages"`
-	SrcPackages       SrcPackages            `json:",omitempty"`
-	EnabledDnfModules []string               `json:"enabledDnfModules,omitempty"` // for dnf modules
-	WordPressPackages WordPressPackages      `json:",omitempty"`
-	LibraryScanners   LibraryScanners        `json:"libraries,omitempty"`
-	CweDict           CweDict                `json:"cweDict,omitempty"`
-	Optional          map[string]interface{} `json:",omitempty"`
+	ScannedCves       VulnInfos                `json:"scannedCves"`
+	RunningKernel     Kernel                   `json:"runningKernel"`
+	Packages          Packages                 `json:"packages"`
+	SrcPackages       SrcPackages              `json:",omitempty"`
+	EnabledDnfModules []string                 `json:"enabledDnfModules,omitempty"` // for dnf modules
+	WordPressPackages WordPressPackages        `json:",omitempty"`
+	GitHubManifests   DependencyGraphManifests `json:"gitHubManifests,omitempty"`
+	LibraryScanners   LibraryScanners          `json:"libraries,omitempty"`
+	CweDict           CweDict                  `json:"cweDict,omitempty"`
+	Optional          map[string]interface{}   `json:",omitempty"`
 	Config            struct {
 		Scan   config.Config `json:"scan"`
 		Report config.Config `json:"report"`