Skip to content

Update trivy from 0.35.0 to 0.49.1 #1806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 81 commits into from
Feb 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
81 commits
Select commit Hold shift + click to select a range
786584f
Update trivy 0.35.0->0.48.0
shino Feb 27, 2024
0c53fd7
move javadb init to scan.go
shino Dec 8, 2023
2faf688
Add options for java db init()
shino Dec 8, 2023
134b6e7
Update scanner/base.go
shino Dec 11, 2023
09a71a4
Remove unused codes
shino Dec 11, 2023
816fb0e
Add some lock file names
shino Dec 11, 2023
77446e1
Typo fix
shino Dec 11, 2023
e62f052
Remove space character (0x20)
shino Dec 11, 2023
13f59a6
Add java-db options for integration scan
shino Dec 11, 2023
a783d3d
Minor fomartting fix
shino Dec 12, 2023
1f450db
minor fix
shino Dec 12, 2023
3776b93
conda is NOT supported by Trivy for library scan
shino Dec 12, 2023
bb90b2f
Configure trivy log in report command too
shino Dec 12, 2023
a4f5bdf
Init trivy in scanner
shino Dec 14, 2023
766fc91
Use trivy's jar.go and replace client which does almost nothing
shino Dec 14, 2023
40179f2
mv jar.go
shino Dec 15, 2023
4f45caa
Add sha1 hash to result and add filepath for report phase
shino Dec 15, 2023
db1c9c1
Undo added 'vuls scan' options
shino Dec 15, 2023
c675d10
Update oras-go to 1.2.4
shino Feb 2, 2024
b798f0f
Move Java DB related config items to report side
shino Feb 4, 2024
34883d7
Add java db search in detect phase
shino Feb 4, 2024
956540d
filter top level jar only
shino Feb 7, 2024
657e8d0
Update trivy to 0.49.1
shino Feb 8, 2024
44653bc
go mod tidy
shino Feb 27, 2024
60c66fe
Update to newer interface
shino Feb 8, 2024
e21f0df
Refine lock file list, h/t MaineK00n
shino Feb 11, 2024
93346d0
Avoid else clauses if possible, h/t MaineK00n
shino Feb 11, 2024
32ed516
Avoid missing word for find and lang types, h/t MaineK00n
shino Feb 11, 2024
7c22963
Add missing ecosystems, h/t MaineK00n
shino Feb 11, 2024
a8243a5
Add comments why to use custom jar analyzer, h/t MaineK00n
shino Feb 11, 2024
1f3d67e
Misc
shino Feb 11, 2024
26c2884
Misc
shino Feb 11, 2024
9c06b42
Misc
shino Feb 11, 2024
e65c3e9
Include go-dep-parser's pares.go for modification
shino Feb 13, 2024
1b4785b
Move digest field from LibraryScanner to Library
shino Feb 13, 2024
69a83ac
Use inner jars sha1 for each
shino Feb 13, 2024
f377f92
Add Seek to file head before handling zip file entry
shino Feb 13, 2024
87f866f
Leave Digest feild empty for entries from pom.xml
shino Feb 13, 2024
d856f7b
Don't import python/pkg (don't look into package.json)
shino Feb 14, 2024
db0fbca
Make privete where private is sufficient
shino Feb 14, 2024
3519b9e
Remove duplicate after Java DB lookup
shino Feb 14, 2024
1558757
misc
shino Feb 14, 2024
eb49c53
go mod tidy
shino Feb 27, 2024
ee3bf78
Comment out ruby/gemspec
shino Feb 14, 2024
7cb613f
misc
shino Feb 14, 2024
f23e7d2
Comment out python/packaging
shino Feb 15, 2024
360a472
misc
shino Feb 15, 2024
0341959
Use custom jar
shino Feb 15, 2024
8081ccf
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
af58616
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
7b0424b
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
e539e51
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
075b6d8
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
9db1c91
Update scanner/trivy/jar/jar.go
shino Feb 16, 2024
2cbf7d6
Update detector/library.go
shino Feb 16, 2024
02ae6da
Update models/library.go
shino Feb 16, 2024
9008339
Update scanner/base.go
shino Feb 16, 2024
b7946cf
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
b87139a
Update scanner/trivy/jar/parse.go
shino Feb 16, 2024
f86775c
Missing changes in name change
shino Feb 16, 2024
6142d63
Update models/github.go
shino Feb 21, 2024
ab63cd7
Update models/library.go
shino Feb 21, 2024
a436aca
Update models/library.go
shino Feb 21, 2024
3f0a465
Update models/library.go
shino Feb 21, 2024
0e9e1af
Update scanner/base.go
shino Feb 21, 2024
b19defe
Update scanner/base.go
shino Feb 21, 2024
312846f
Update scanner/trivy/jar/jar.go
shino Feb 21, 2024
2e98413
Don't import fanal/types at github.go
shino Feb 22, 2024
7de8810
Rewrite code around java db initialization
shino Feb 22, 2024
378f3c0
Add comment
shino Feb 22, 2024
35b2420
refactor
shino Feb 26, 2024
296e5df
Close java db client
shino Feb 26, 2024
e6dd31a
rename
shino Feb 26, 2024
a5226a1
Let LibraryScanner have java db client
shino Feb 27, 2024
57438f8
Update detector/library.go
shino Feb 27, 2024
27b86ae
Update detector/library.go
shino Feb 27, 2024
9879573
Update detector/library.go
shino Feb 27, 2024
6b2e170
Update detector/library.go
shino Feb 27, 2024
e21ca9e
inline variable
shino Feb 27, 2024
aacf08a
misc
shino Feb 27, 2024
0e752b0
Fix typo
shino Feb 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion GNUmakefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
install \
all \
vendor \
lint \
lint \
vet \
fmt \
fmtcheck \
Expand Down
10 changes: 9 additions & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ type ScanOpts struct {
type ReportOpts struct {
CvssScoreOver float64 `json:"cvssScoreOver,omitempty"`
ConfidenceScoreOver int `json:"confidenceScoreOver,omitempty"`
TrivyCacheDBDir string `json:"trivyCacheDBDir,omitempty"`
NoProgress bool `json:"noProgress,omitempty"`
RefreshCve bool `json:"refreshCve,omitempty"`
IgnoreUnfixed bool `json:"ignoreUnfixed,omitempty"`
Expand All @@ -85,6 +84,15 @@ type ReportOpts struct {
DiffMinus bool `json:"diffMinus,omitempty"`
Diff bool `json:"diff,omitempty"`
Lang string `json:"lang,omitempty"`

TrivyOpts
}

// TrivyOpts is options for trivy DBs
type TrivyOpts struct {
TrivyCacheDBDir string `json:"trivyCacheDBDir,omitempty"`
TrivyJavaDBRepository string `json:"trivyJavaDBRepository,omitempty"`
TrivySkipJavaDBUpdate bool `json:"trivySkipJavaDBUpdate,omitempty"`
}

// ValidateOnConfigtest validates
Expand Down
16 changes: 8 additions & 8 deletions contrib/future-vuls/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
- upload vuls results json to future-vuls

- `future-vuls discover`
- Explore hosts within the CIDR range using the ping command
- Describe the information including CPE on the found hosts in a toml-formatted file.
- Exec snmp2cpe(https://github.com/future-architect/vuls/pull/1625) to active hosts to obtain CPE<br>
Commands running internally  `snmp2cpe v2c {IPAddr} public | snmp2cpe convert`<br>
- Explores hosts within the CIDR range using the ping command
- Describes the information including CPEs on the found hosts in a toml-formatted file
- Executes snmp2cpe(https://github.com/future-architect/vuls/pull/1625) to active hosts to obtain CPE,
Commands running internally `snmp2cpe v2c {IPAddr} public | snmp2cpe convert`

Structure of toml-formatted file
```
[server.{ip}]
Expand All @@ -23,12 +23,12 @@ fvuls_sync = false

- `future-vuls add-cpe`
- Create pseudo server to Fvuls to obtain uuid and Upload CPE information on the specified(FvulsSync is true and UUID is obtained) hosts to Fvuls
- Fvuls_Sync must be rewritten to true to designate it as the target of the command<br><br>
- Fvuls_Sync must be rewritten to true to designate it as the target of the command


1. `future-vuls discover`
1. `future-vuls discover`

2. `future-vuls add-cpe`
2. `future-vuls add-cpe`

These two commands are used to manage the CPE of network devices, and by executing the commands in the order from the top, you can manage the CPE of each device in Fvuls

Expand Down
13 changes: 5 additions & 8 deletions contrib/trivy/parser/v2/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,13 @@ func setScanResultMeta(scanResult *models.ScanResult, report *types.Report) erro
scanResult.ServerName = report.ArtifactName
if report.ArtifactType == "container_image" {
matches := dockerTagPattern.FindStringSubmatch(report.ArtifactName)
var imageName, imageTag string
// initial values are for without image tag
var imageName = report.ArtifactName
var imageTag = "latest" // Complement if the tag is omitted
if 2 < len(matches) {
// including the image tag
imageName = matches[1]
imageTag = matches[2]
} else {
// no image tag
imageName = report.ArtifactName
imageTag = "latest" // Complement if the tag is omitted
}
scanResult.ServerName = fmt.Sprintf("%s:%s", imageName, imageTag)
if scanResult.Optional == nil {
Expand All @@ -64,11 +62,10 @@ func setScanResultMeta(scanResult *models.ScanResult, report *types.Report) erro
scanResult.Optional["TRIVY_IMAGE_TAG"] = imageTag
}

scanResult.Family = constant.ServerTypePseudo
if report.Metadata.OS != nil {
scanResult.Family = report.Metadata.OS.Family
scanResult.Family = string(report.Metadata.OS.Family)
scanResult.Release = report.Metadata.OS.Name
} else {
scanResult.Family = constant.ServerTypePseudo
}

scanResult.ScannedAt = time.Now()
Expand Down
42 changes: 22 additions & 20 deletions contrib/trivy/pkg/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"sort"
"time"

"github.com/aquasecurity/trivy/pkg/fanal/analyzer/os"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/types"

"github.com/future-architect/vuls/models"
Expand Down Expand Up @@ -92,7 +92,7 @@ func Convert(results types.Results) (result *models.ScanResult, err error) {
})
} else {
vulnInfo.LibraryFixedIns = append(vulnInfo.LibraryFixedIns, models.LibraryFixedIn{
Key: trivyResult.Type,
Key: string(trivyResult.Type),
Name: vuln.PkgName,
Path: trivyResult.Target,
FixedIn: vuln.FixedVersion,
Expand Down Expand Up @@ -190,24 +190,26 @@ func Convert(results types.Results) (result *models.ScanResult, err error) {
return scanResult, nil
}

func isTrivySupportedOS(family string) bool {
supportedFamilies := map[string]struct{}{
os.RedHat: {},
os.Debian: {},
os.Ubuntu: {},
os.CentOS: {},
os.Rocky: {},
os.Alma: {},
os.Fedora: {},
os.Amazon: {},
os.Oracle: {},
os.Windows: {},
os.OpenSUSE: {},
os.OpenSUSELeap: {},
os.OpenSUSETumbleweed: {},
os.SLES: {},
os.Photon: {},
os.Alpine: {},
func isTrivySupportedOS(family ftypes.TargetType) bool {
supportedFamilies := map[ftypes.TargetType]struct{}{
ftypes.Alma: {},
ftypes.Alpine: {},
ftypes.Amazon: {},
ftypes.CBLMariner: {},
ftypes.CentOS: {},
ftypes.Chainguard: {},
ftypes.Debian: {},
ftypes.Fedora: {},
ftypes.OpenSUSE: {},
ftypes.OpenSUSELeap: {},
ftypes.OpenSUSETumbleweed: {},
ftypes.Oracle: {},
ftypes.Photon: {},
ftypes.RedHat: {},
ftypes.Rocky: {},
ftypes.SLES: {},
ftypes.Ubuntu: {},
ftypes.Wolfi: {},
}
_, ok := supportedFamilies[family]
return ok
Expand Down
2 changes: 1 addition & 1 deletion detector/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func Detect(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
r.ScannedCves = models.VulnInfos{}
}

if err := DetectLibsCves(&r, config.Conf.TrivyCacheDBDir, config.Conf.NoProgress); err != nil {
if err := DetectLibsCves(&r, config.Conf.TrivyOpts, config.Conf.LogOpts, config.Conf.NoProgress); err != nil {
return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err)
}

Expand Down
48 changes: 32 additions & 16 deletions detector/library.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,40 +9,56 @@ import (
trivydb "github.com/aquasecurity/trivy-db/pkg/db"
"github.com/aquasecurity/trivy-db/pkg/metadata"
"github.com/aquasecurity/trivy/pkg/db"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/javadb"
"github.com/aquasecurity/trivy/pkg/log"
"golang.org/x/xerrors"

"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/models"
)

// DetectLibsCves fills LibraryScanner information
func DetectLibsCves(r *models.ScanResult, cacheDir string, noProgress bool) (err error) {
func DetectLibsCves(r *models.ScanResult, trivyOpts config.TrivyOpts, logOpts logging.LogOpts, noProgress bool) (err error) {
totalCnt := 0
if len(r.LibraryScanners) == 0 {
return
}

// initialize trivy's logger and db
err = log.InitLogger(false, false)
err = log.InitLogger(logOpts.Debug, logOpts.Quiet)
if err != nil {
return err
return xerrors.Errorf("Failed to init trivy logger. err: %w", err)
}

logging.Log.Info("Updating library db...")
if err := downloadDB("", cacheDir, noProgress, false); err != nil {
return err
if err := downloadDB("", trivyOpts, noProgress, false); err != nil {
return xerrors.Errorf("Failed to download trivy DB. err: %w", err)
}

if err := trivydb.Init(cacheDir); err != nil {
return err
if err := trivydb.Init(trivyOpts.TrivyCacheDBDir); err != nil {
return xerrors.Errorf("Failed to init trivy DB. err: %w", err)
}
defer trivydb.Close()

var javaDBClient *javadb.DB
defer javaDBClient.Close()
for _, lib := range r.LibraryScanners {
if lib.Type == ftypes.Jar {
if javaDBClient == nil {
javadb.Init(trivyOpts.TrivyCacheDBDir, trivyOpts.TrivyJavaDBRepository, trivyOpts.TrivySkipJavaDBUpdate, noProgress, ftypes.RegistryOptions{})

javaDBClient, err = javadb.NewClient()
if err != nil {
return xerrors.Errorf("Failed to download or open trivy Java DB. err: %w", err)
}
}
lib.JavaDBClient = javaDBClient
}

vinfos, err := lib.Scan()
if err != nil {
return err
return xerrors.Errorf("Failed to scan library. err: %w", err)
}
for _, vinfo := range vinfos {
vinfo.Confidences.AppendIfMissing(models.TrivyMatch)
Expand All @@ -62,8 +78,8 @@ func DetectLibsCves(r *models.ScanResult, cacheDir string, noProgress bool) (err
return nil
}

func downloadDB(appVersion, cacheDir string, quiet, skipUpdate bool) error {
client := db.NewClient(cacheDir, quiet, false)
func downloadDB(appVersion string, trivyOpts config.TrivyOpts, noProgress, skipUpdate bool) error {
client := db.NewClient(trivyOpts.TrivyCacheDBDir, noProgress)
ctx := context.Background()
needsUpdate, err := client.NeedsUpdate(appVersion, skipUpdate)
if err != nil {
Expand All @@ -73,14 +89,14 @@ func downloadDB(appVersion, cacheDir string, quiet, skipUpdate bool) error {
if needsUpdate {
logging.Log.Info("Need to update DB")
logging.Log.Info("Downloading DB...")
if err := client.Download(ctx, cacheDir); err != nil {
return xerrors.Errorf("failed to download vulnerability DB: %w", err)
if err := client.Download(ctx, trivyOpts.TrivyCacheDBDir, ftypes.RegistryOptions{}); err != nil {
return xerrors.Errorf("Failed to download vulnerability DB. err: %w", err)
}
}

// for debug
if err := showDBInfo(cacheDir); err != nil {
return xerrors.Errorf("failed to show database info: %w", err)
if err := showDBInfo(trivyOpts.TrivyCacheDBDir); err != nil {
return xerrors.Errorf("Failed to show database info. err: %w", err)
}
return nil
}
Expand All @@ -89,7 +105,7 @@ func showDBInfo(cacheDir string) error {
m := metadata.NewClient(cacheDir)
meta, err := m.Get()
if err != nil {
return xerrors.Errorf("something wrong with DB: %w", err)
return xerrors.Errorf("Failed to get DB metadata. err: %w", err)
}
log.Logger.Debugf("DB Schema: %d, UpdatedAt: %s, NextUpdate: %s, DownloadedAt: %s",
meta.Version, meta.UpdatedAt, meta.NextUpdate, meta.DownloadedAt)
Expand Down
Loading