From 1071ab759c9feec0e0d99fd30749c323be01eb80 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Mon, 4 Mar 2024 16:45:40 +0900 Subject: [PATCH 01/14] Avoid to use once inside trivy javadb Updater Because detector package may be used as library-like way --- detector/javadb/javadb.go | 100 ++++++++++++++++++++++++++++++++++++++ detector/library.go | 14 +++--- models/library.go | 4 +- 3 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 detector/javadb/javadb.go diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go new file mode 100644 index 0000000000..9f14b640c9 --- /dev/null +++ b/detector/javadb/javadb.go @@ -0,0 +1,100 @@ +//go:build !scanner +// +build !scanner + +package javadb + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/aquasecurity/go-dep-parser/pkg/java/jar" + "github.com/aquasecurity/trivy-java-db/pkg/db" + "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/oci" + "golang.org/x/xerrors" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/logging" +) + +func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { + repo := fmt.Sprintf("%s:%d", trivyOpts.TrivyJavaDBRepository, db.SchemaVersion) + dbDir := filepath.Join(trivyOpts.TrivyCacheDBDir, "java-db") + + metac := db.NewMetadata(dbDir) + meta, err := metac.Get() + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return xerrors.Errorf("Failed to get Java DB metadata. err: %w", err) + } else if trivyOpts.TrivySkipJavaDBUpdate { + logging.Log.Error("Could not skip, the first run cannot skip downloading Java DB") + return xerrors.New("'--skip-java-db-update' cannot be specified on the first run") + } + } + + if (meta.Version != db.SchemaVersion || meta.NextUpdate.Before(time.Now().UTC())) && !trivyOpts.TrivySkipJavaDBUpdate { + // Download DB + logging.Log.Infof("Trivy Java DB Repository: %s", repo) + logging.Log.Info("Downloading Trivy Java DB...") + + var a *oci.Artifact + if a, err = oci.NewArtifact(repo, noProgress, types.RegistryOptions{}); err != nil { + return xerrors.Errorf("oci error: %w", err) + } + if err = a.Download(context.Background(), dbDir, oci.DownloadOption{MediaType: "application/vnd.aquasec.trivy.javadb.layer.v1.tar+gzip"}); err != nil { + return xerrors.Errorf("Failed to download Trivy Java DB. err: %w", err) + } + + // Parse the newly downloaded metadata.json + meta, err = metac.Get() + if err != nil { + return xerrors.Errorf("Failed to get Trivy Java DB metadata. err: %w", err) + } + + // Update DownloadedAt + meta.DownloadedAt = time.Now().UTC() + if err = metac.Update(meta); err != nil { + return xerrors.Errorf("Failed to update Trivy Java DB metadata. erro: %w", err) + } + } + + return nil +} + +type DBClient struct { + driver db.DB +} + +func NewClient(cacheDBDir string) (*DBClient, error) { + driver, err := db.New(filepath.Join(cacheDBDir, "java-db")) + if err != nil { + return nil, xerrors.Errorf("Failed to open Trivy Java DB. err: %w", err) + } + return &DBClient{driver: driver}, nil +} + +func (client *DBClient) Close() error { + if client == nil { + return nil + } + + return client.driver.Close() +} + +func (client *DBClient) SearchBySHA1(sha1 string) (jar.Properties, error) { + index, err := client.driver.SelectIndexBySha1(sha1) + if err != nil { + return jar.Properties{}, xerrors.Errorf("Failed to select from Trivy Java DB. err: %w", err) + } else if index.ArtifactID == "" { + return jar.Properties{}, xerrors.Errorf("digest %s: %w", sha1, jar.ArtifactNotFoundErr) + } + return jar.Properties{ + GroupID: index.GroupID, + ArtifactID: index.ArtifactID, + Version: index.Version, + }, nil +} diff --git a/detector/library.go b/detector/library.go index 04f0a4fd1c..e2575e12fd 100644 --- a/detector/library.go +++ b/detector/library.go @@ -10,11 +10,11 @@ import ( "github.com/aquasecurity/trivy-db/pkg/metadata" "github.com/aquasecurity/trivy/pkg/db" ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" - "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/log" "golang.org/x/xerrors" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/detector/javadb" "github.com/future-architect/vuls/logging" "github.com/future-architect/vuls/models" ) @@ -41,16 +41,18 @@ func DetectLibsCves(r *models.ScanResult, trivyOpts config.TrivyOpts, logOpts lo } defer trivydb.Close() - var javaDBClient *javadb.DB + var javaDBClient *javadb.DBClient defer javaDBClient.Close() for _, lib := range r.LibraryScanners { if lib.Type == ftypes.Jar { if javaDBClient == nil { - javadb.Init(trivyOpts.TrivyCacheDBDir, trivyOpts.TrivyJavaDBRepository, trivyOpts.TrivySkipJavaDBUpdate, noProgress, ftypes.RegistryOptions{}) + if err := javadb.UpdateJavaDB(trivyOpts, noProgress); err != nil { + return xerrors.Errorf("Failed to update Trivy Java DB. err: %w", err) + } - javaDBClient, err = javadb.NewClient() + javaDBClient, err = javadb.NewClient(trivyOpts.TrivyCacheDBDir) if err != nil { - return xerrors.Errorf("Failed to download or open trivy Java DB. err: %w", err) + return xerrors.Errorf("Failed to open Trivy Java DB. err: %w", err) } } lib.JavaDBClient = javaDBClient @@ -107,7 +109,7 @@ func showDBInfo(cacheDir string) error { if err != nil { return xerrors.Errorf("Failed to get DB metadata. err: %w", err) } - log.Logger.Debugf("DB Schema: %d, UpdatedAt: %s, NextUpdate: %s, DownloadedAt: %s", + logging.Log.Debugf("DB Schema: %d, UpdatedAt: %s, NextUpdate: %s, DownloadedAt: %s", meta.Version, meta.UpdatedAt, meta.NextUpdate, meta.DownloadedAt) return nil } diff --git a/models/library.go b/models/library.go index 970068d401..850dfc81a0 100644 --- a/models/library.go +++ b/models/library.go @@ -10,11 +10,11 @@ import ( trivyDBTypes "github.com/aquasecurity/trivy-db/pkg/types" "github.com/aquasecurity/trivy/pkg/detector/library" ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" - "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/types" "github.com/samber/lo" "golang.org/x/xerrors" + "github.com/future-architect/vuls/detector/javadb" "github.com/future-architect/vuls/logging" ) @@ -51,7 +51,7 @@ type LibraryScanner struct { // The path to the Lockfile is stored. LockfilePath string `json:"path,omitempty"` - JavaDBClient *javadb.DB `json:"-"` + JavaDBClient *javadb.DBClient `json:"-"` } // Library holds the attribute of a package library From 53d3769df0ac8fed3ca011e283d9ad009a787c9a Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:18:27 +0900 Subject: [PATCH 02/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 9f14b640c9..761536b285 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -58,7 +58,7 @@ func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { // Update DownloadedAt meta.DownloadedAt = time.Now().UTC() if err = metac.Update(meta); err != nil { - return xerrors.Errorf("Failed to update Trivy Java DB metadata. erro: %w", err) + return xerrors.Errorf("Failed to update Trivy Java DB metadata. err: %w", err) } } From e8a313f1b7693cde667598648908d94be08a1f2b Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:18:47 +0900 Subject: [PATCH 03/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 761536b285..4c6520dbe8 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -38,6 +38,7 @@ func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { if (meta.Version != db.SchemaVersion || meta.NextUpdate.Before(time.Now().UTC())) && !trivyOpts.TrivySkipJavaDBUpdate { // Download DB + repo := fmt.Sprintf("%s:%d", trivyOpts.TrivyJavaDBRepository, db.SchemaVersion) logging.Log.Infof("Trivy Java DB Repository: %s", repo) logging.Log.Info("Downloading Trivy Java DB...") From c91fee937da9129f9db515ca48b734930c684c92 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:19:19 +0900 Subject: [PATCH 04/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 4c6520dbe8..62e2992e90 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -44,7 +44,7 @@ func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { var a *oci.Artifact if a, err = oci.NewArtifact(repo, noProgress, types.RegistryOptions{}); err != nil { - return xerrors.Errorf("oci error: %w", err) + return xerrors.Errorf("Failed to new oci artifact. err: %w", err) } if err = a.Download(context.Background(), dbDir, oci.DownloadOption{MediaType: "application/vnd.aquasec.trivy.javadb.layer.v1.tar+gzip"}); err != nil { return xerrors.Errorf("Failed to download Trivy Java DB. err: %w", err) From 92d6ab1e71bd38b26ce37f07f298a2d163cc8101 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:19:38 +0900 Subject: [PATCH 05/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 62e2992e90..583c86d303 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -21,6 +21,7 @@ import ( "github.com/future-architect/vuls/logging" ) +// UpdateJavaDB updates Trivy Java DB func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { repo := fmt.Sprintf("%s:%d", trivyOpts.TrivyJavaDBRepository, db.SchemaVersion) dbDir := filepath.Join(trivyOpts.TrivyCacheDBDir, "java-db") From 2c8b5bc6c64799b03978ec2e56d37de0909a1c8c Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:19:46 +0900 Subject: [PATCH 06/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 583c86d303..2ce308faca 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -71,6 +71,7 @@ type DBClient struct { driver db.DB } +// NewClient returns Trivy Java DB Client func NewClient(cacheDBDir string) (*DBClient, error) { driver, err := db.New(filepath.Join(cacheDBDir, "java-db")) if err != nil { From 1eacb10851da29f231bd360545729fc2a3f62c35 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:19:53 +0900 Subject: [PATCH 07/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 2ce308faca..e9348e84f1 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -80,6 +80,7 @@ func NewClient(cacheDBDir string) (*DBClient, error) { return &DBClient{driver: driver}, nil } +// Close closes Trivy Java DB Client func (client *DBClient) Close() error { if client == nil { return nil From a05d2dee86ed4b4487ff4d80238fe20b57d9ea4b Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:20:14 +0900 Subject: [PATCH 08/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index e9348e84f1..6d00af6bf6 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -89,6 +89,7 @@ func (client *DBClient) Close() error { return client.driver.Close() } +// SearchBySHA1 searches Jar Property by SHA1 func (client *DBClient) SearchBySHA1(sha1 string) (jar.Properties, error) { index, err := client.driver.SelectIndexBySha1(sha1) if err != nil { From 538ad51c9a331086818ce72de9011e5ca35ba86c Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:20:27 +0900 Subject: [PATCH 09/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 6d00af6bf6..0956aa858f 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -67,6 +67,7 @@ func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { return nil } +// DBClient is Trivy Java DB Client type DBClient struct { driver db.DB } From 2923a7cf29a369b1440c42bcd221effd68e74494 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:20:56 +0900 Subject: [PATCH 10/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 0956aa858f..8f5fc514a2 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -23,7 +23,6 @@ import ( // UpdateJavaDB updates Trivy Java DB func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { - repo := fmt.Sprintf("%s:%d", trivyOpts.TrivyJavaDBRepository, db.SchemaVersion) dbDir := filepath.Join(trivyOpts.TrivyCacheDBDir, "java-db") metac := db.NewMetadata(dbDir) From c9271cbd1df5d19881f0b806eab0a74a1244db8d Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:21:58 +0900 Subject: [PATCH 11/14] Update detector/javadb/javadb.go Co-authored-by: MaineK00n --- detector/javadb/javadb.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 8f5fc514a2..0e2f6ff0c5 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -94,8 +94,9 @@ func (client *DBClient) SearchBySHA1(sha1 string) (jar.Properties, error) { index, err := client.driver.SelectIndexBySha1(sha1) if err != nil { return jar.Properties{}, xerrors.Errorf("Failed to select from Trivy Java DB. err: %w", err) - } else if index.ArtifactID == "" { - return jar.Properties{}, xerrors.Errorf("digest %s: %w", sha1, jar.ArtifactNotFoundErr) + } + if index.ArtifactID == "" { + return jar.Properties{}, xerrors.Errorf("Failed to search ArtifactID by digest %s. err: %w", sha1, jar.ArtifactNotFoundErr) } return jar.Properties{ GroupID: index.GroupID, From fb53370d1b090a652ac7094d5ab822798c06e45e Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:27:22 +0900 Subject: [PATCH 12/14] Avoid else if, unless necessary --- detector/javadb/javadb.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index 0e2f6ff0c5..eb2a3c28ef 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -30,9 +30,10 @@ func UpdateJavaDB(trivyOpts config.TrivyOpts, noProgress bool) error { if err != nil { if !errors.Is(err, os.ErrNotExist) { return xerrors.Errorf("Failed to get Java DB metadata. err: %w", err) - } else if trivyOpts.TrivySkipJavaDBUpdate { + } + if trivyOpts.TrivySkipJavaDBUpdate { logging.Log.Error("Could not skip, the first run cannot skip downloading Java DB") - return xerrors.New("'--skip-java-db-update' cannot be specified on the first run") + return xerrors.New("'--trivy-skip-java-db-update' cannot be specified on the first run") } } From a118ac2243c2fe5a8f1f702d16197b1df5a4e571 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 09:45:19 +0900 Subject: [PATCH 13/14] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1692dfea5e..e13dda2d01 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/aquasecurity/go-dep-parser v0.0.0-20240202105001-4f19ab402b0b github.com/aquasecurity/trivy v0.49.1 github.com/aquasecurity/trivy-db v0.0.0-20231005141211-4fc651f7ac8d + github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go v1.49.21 github.com/c-robinson/iplib v1.0.8 @@ -107,7 +108,6 @@ require ( github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect github.com/aquasecurity/trivy-iac v0.8.0 // indirect - github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48 // indirect github.com/aquasecurity/trivy-policies v0.8.0 // indirect github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.26.3 // indirect From a12615625adbb9e85f1fcf65afed1326b4e299c4 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 5 Mar 2024 10:05:13 +0900 Subject: [PATCH 14/14] Add package comment --- detector/javadb/javadb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/detector/javadb/javadb.go b/detector/javadb/javadb.go index eb2a3c28ef..e01f3d3b04 100644 --- a/detector/javadb/javadb.go +++ b/detector/javadb/javadb.go @@ -1,6 +1,7 @@ //go:build !scanner // +build !scanner +// Package javadb implements functions that wrap trivy-java-db module. package javadb import (