Skip to content

Commit 1d97e91

Browse files
kotakanbekotakanbe
authored
fix(libscan): delete map that keeps all file contents detected by FindLock to save memory (#1556)
* fix(libscan): delete Map that keeps all files detected by FindLock to save memory * continue analyzing libs if err occurred * FindLockDirs * fix * fix
1 parent 96333f3 commit 1d97e91

File tree

6 files changed

+134
-119
lines changed

6 files changed

+134
-119
lines changed

‎config/config.go

+1
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ type ServerInfo struct {
240240
Optional map[string]interface{} `toml:"optional,omitempty" json:"optional,omitempty"` // Optional key-value set that will be outputted to JSON
241241
Lockfiles []string `toml:"lockfiles,omitempty" json:"lockfiles,omitempty"` // ie) path/to/package-lock.json
242242
FindLock bool `toml:"findLock,omitempty" json:"findLock,omitempty"`
243+
FindLockDirs []string `toml:"findLockDirs,omitempty" json:"findLockDirs,omitempty"`
243244
Type string `toml:"type,omitempty" json:"type,omitempty"` // "pseudo" or ""
244245
IgnoredJSONKeys []string `toml:"ignoredJSONKeys,omitempty" json:"ignoredJSONKeys,omitempty"`
245246
WordPress *WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"`

‎logging/logutil.go

+7
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,13 @@ func NewNormalLogger() Logger {
4545
return Logger{Entry: logrus.Entry{Logger: logrus.New()}}
4646
}
4747

48+
// NewNormalLogger creates normal logger
49+
func NewIODiscardLogger() Logger {
50+
l := logrus.New()
51+
l.Out = io.Discard
52+
return Logger{Entry: logrus.Entry{Logger: l}}
53+
}
54+
4855
// NewCustomLogger creates logrus
4956
func NewCustomLogger(debug, quiet, logToFile bool, logDir, logMsgAnsiColor, serverName string) Logger {
5057
log := logrus.New()

‎scanner/base.go

+124-117
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,6 @@ func (l *base) detectPlatform() {
361361

362362
//TODO Azure, GCP...
363363
l.setPlatform(models.Platform{Name: "other"})
364-
return
365364
}
366365

367366
var dsFingerPrintPrefix = "AgentStatus.agentCertHash: "
@@ -582,12 +581,6 @@ func (l *base) parseSystemctlStatus(stdout string) string {
582581
return ss[1]
583582
}
584583

585-
// LibFile : library file content
586-
type LibFile struct {
587-
Contents []byte
588-
Filemode os.FileMode
589-
}
590-
591584
func (l *base) scanLibraries() (err error) {
592585
if len(l.LibraryScanners) != 0 {
593586
return nil
@@ -598,9 +591,9 @@ func (l *base) scanLibraries() (err error) {
598591
return nil
599592
}
600593

601-
l.log.Info("Scanning Lockfile...")
594+
l.log.Info("Scanning Language-specific Packages...")
602595

603-
libFilemap := map[string]LibFile{}
596+
found := map[string]bool{}
604597
detectFiles := l.ServerInfo.Lockfiles
605598

606599
priv := noSudo
@@ -615,9 +608,17 @@ func (l *base) scanLibraries() (err error) {
615608
findopt += fmt.Sprintf("-name %q -o ", filename)
616609
}
617610

611+
dir := "/"
612+
if len(l.ServerInfo.FindLockDirs) != 0 {
613+
dir = strings.Join(l.ServerInfo.FindLockDirs, " ")
614+
} else {
615+
l.log.Infof("It's recommended to specify FindLockDirs in config.toml. If FindLockDirs is not specified, all directories under / will be searched, which may increase CPU load")
616+
}
617+
l.log.Infof("Finding files under %s", dir)
618+
618619
// delete last "-o "
619620
// find / -type f -and \( -name "package-lock.json" -o -name "yarn.lock" ... \) 2>&1 | grep -v "find: "
620-
cmd := fmt.Sprintf(`find / -type f -and \( ` + findopt[:len(findopt)-3] + ` \) 2>&1 | grep -v "find: "`)
621+
cmd := fmt.Sprintf(`find %s -type f -and \( `+findopt[:len(findopt)-3]+` \) 2>&1 | grep -v "find: "`, dir)
621622
r := exec(l.ServerInfo, cmd, priv)
622623
if r.ExitStatus != 0 && r.ExitStatus != 1 {
623624
return xerrors.Errorf("Failed to find lock files")
@@ -635,116 +636,62 @@ func (l *base) scanLibraries() (err error) {
635636
}
636637

637638
// skip already exist
638-
if _, ok := libFilemap[path]; ok {
639+
if _, ok := found[path]; ok {
639640
continue
640641
}
641642

642-
var f LibFile
643+
var contents []byte
644+
var filemode os.FileMode
645+
643646
switch l.Distro.Family {
644647
case constant.ServerTypePseudo:
645648
fileinfo, err := os.Stat(path)
646649
if err != nil {
647-
return xerrors.Errorf("Failed to get target file info. err: %w, filepath: %s", err, path)
650+
l.log.Warnf("Failed to get target file info. err: %s, filepath: %s", err, path)
651+
continue
648652
}
649-
f.Filemode = fileinfo.Mode().Perm()
650-
f.Contents, err = os.ReadFile(path)
653+
filemode = fileinfo.Mode().Perm()
654+
contents, err = os.ReadFile(path)
651655
if err != nil {
652-
return xerrors.Errorf("Failed to read target file contents. err: %w, filepath: %s", err, path)
656+
l.log.Warnf("Failed to read target file contents. err: %s, filepath: %s", err, path)
657+
continue
653658
}
654659
default:
660+
l.log.Debugf("Analyzing file: %s", path)
655661
cmd := fmt.Sprintf(`stat -c "%%a" %s`, path)
656-
r := exec(l.ServerInfo, cmd, priv)
662+
r := exec(l.ServerInfo, cmd, priv, logging.NewIODiscardLogger())
657663
if !r.isSuccess() {
658-
return xerrors.Errorf("Failed to get target file permission: %s, filepath: %s", r, path)
664+
l.log.Warnf("Failed to get target file permission: %s, filepath: %s", r, path)
665+
continue
659666
}
660667
permStr := fmt.Sprintf("0%s", strings.ReplaceAll(r.Stdout, "\n", ""))
661668
perm, err := strconv.ParseUint(permStr, 8, 32)
662669
if err != nil {
663-
return xerrors.Errorf("Failed to parse permission string. err: %w, permission string: %s", err, permStr)
670+
l.log.Warnf("Failed to parse permission string. err: %s, permission string: %s", err, permStr)
671+
continue
664672
}
665-
f.Filemode = os.FileMode(perm)
673+
filemode = os.FileMode(perm)
666674

667675
cmd = fmt.Sprintf("cat %s", path)
668-
r = exec(l.ServerInfo, cmd, priv)
676+
r = exec(l.ServerInfo, cmd, priv, logging.NewIODiscardLogger())
669677
if !r.isSuccess() {
670-
return xerrors.Errorf("Failed to get target file contents: %s, filepath: %s", r, path)
678+
l.log.Warnf("Failed to get target file contents: %s, filepath: %s", r, path)
679+
continue
671680
}
672-
f.Contents = []byte(r.Stdout)
681+
contents = []byte(r.Stdout)
673682
}
674-
libFilemap[path] = f
675-
}
676-
677-
var libraryScanners []models.LibraryScanner
678-
if libraryScanners, err = AnalyzeLibraries(context.Background(), libFilemap, l.ServerInfo.Mode.IsOffline()); err != nil {
679-
return err
683+
found[path] = true
684+
var libraryScanners []models.LibraryScanner
685+
if libraryScanners, err = AnalyzeLibrary(context.Background(), path, contents, filemode, l.ServerInfo.Mode.IsOffline()); err != nil {
686+
return err
687+
}
688+
l.LibraryScanners = append(l.LibraryScanners, libraryScanners...)
680689
}
681-
l.LibraryScanners = append(l.LibraryScanners, libraryScanners...)
682690
return nil
683691
}
684692

685-
// AnalyzeLibraries : detects libs defined in lockfile
686-
func AnalyzeLibraries(ctx context.Context, libFilemap map[string]LibFile, isOffline bool) (libraryScanners []models.LibraryScanner, err error) {
687-
// https://github.com/aquasecurity/trivy/blob/84677903a6fa1b707a32d0e8b2bffc23dde52afa/pkg/fanal/analyzer/const.go
688-
disabledAnalyzers := []analyzer.Type{
689-
// ======
690-
// OS
691-
// ======
692-
analyzer.TypeOSRelease,
693-
analyzer.TypeAlpine,
694-
analyzer.TypeAmazon,
695-
analyzer.TypeCBLMariner,
696-
analyzer.TypeDebian,
697-
analyzer.TypePhoton,
698-
analyzer.TypeCentOS,
699-
analyzer.TypeRocky,
700-
analyzer.TypeAlma,
701-
analyzer.TypeFedora,
702-
analyzer.TypeOracle,
703-
analyzer.TypeRedHatBase,
704-
analyzer.TypeSUSE,
705-
analyzer.TypeUbuntu,
706-
707-
// OS Package
708-
analyzer.TypeApk,
709-
analyzer.TypeDpkg,
710-
analyzer.TypeDpkgLicense,
711-
analyzer.TypeRpm,
712-
analyzer.TypeRpmqa,
713-
714-
// OS Package Repository
715-
analyzer.TypeApkRepo,
716-
717-
// ============
718-
// Image Config
719-
// ============
720-
analyzer.TypeApkCommand,
721-
722-
// =================
723-
// Structured Config
724-
// =================
725-
analyzer.TypeYaml,
726-
analyzer.TypeJSON,
727-
analyzer.TypeDockerfile,
728-
analyzer.TypeTerraform,
729-
analyzer.TypeCloudFormation,
730-
analyzer.TypeHelm,
731-
732-
// ========
733-
// License
734-
// ========
735-
analyzer.TypeLicenseFile,
736-
737-
// ========
738-
// Secrets
739-
// ========
740-
analyzer.TypeSecret,
741-
742-
// =======
743-
// Red Hat
744-
// =======
745-
analyzer.TypeRedHatContentManifestType,
746-
analyzer.TypeRedHatDockerfileType,
747-
}
693+
// AnalyzeLibrary : detects library defined in artifact such as lockfile or jar
694+
func AnalyzeLibrary(ctx context.Context, path string, contents []byte, filemode os.FileMode, isOffline bool) (libraryScanners []models.LibraryScanner, err error) {
748695
anal, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
749696
Group: analyzer.GroupBuiltin,
750697
DisabledAnalyzers: disabledAnalyzers,
@@ -753,34 +700,94 @@ func AnalyzeLibraries(ctx context.Context, libFilemap map[string]LibFile, isOffl
753700
return nil, xerrors.Errorf("Failed to new analyzer group. err: %w", err)
754701
}
755702

756-
for path, f := range libFilemap {
757-
var wg sync.WaitGroup
758-
result := new(analyzer.AnalysisResult)
759-
if err := anal.AnalyzeFile(
760-
ctx,
761-
&wg,
762-
semaphore.NewWeighted(1),
763-
result,
764-
"",
765-
path,
766-
&DummyFileInfo{size: int64(len(f.Contents)), filemode: f.Filemode},
767-
func() (dio.ReadSeekCloserAt, error) { return dio.NopCloser(bytes.NewReader(f.Contents)), nil },
768-
nil,
769-
analyzer.AnalysisOptions{Offline: isOffline},
770-
); err != nil {
771-
return nil, xerrors.Errorf("Failed to get libs. err: %w", err)
772-
}
773-
wg.Wait()
774-
775-
libscan, err := convertLibWithScanner(result.Applications)
776-
if err != nil {
777-
return nil, xerrors.Errorf("Failed to convert libs. err: %w", err)
778-
}
779-
libraryScanners = append(libraryScanners, libscan...)
703+
var wg sync.WaitGroup
704+
result := new(analyzer.AnalysisResult)
705+
if err := anal.AnalyzeFile(
706+
ctx,
707+
&wg,
708+
semaphore.NewWeighted(1),
709+
result,
710+
"",
711+
path,
712+
&DummyFileInfo{size: int64(len(contents)), filemode: filemode},
713+
func() (dio.ReadSeekCloserAt, error) { return dio.NopCloser(bytes.NewReader(contents)), nil },
714+
nil,
715+
analyzer.AnalysisOptions{Offline: isOffline},
716+
); err != nil {
717+
return nil, xerrors.Errorf("Failed to get libs. err: %w", err)
718+
}
719+
wg.Wait()
720+
721+
libscan, err := convertLibWithScanner(result.Applications)
722+
if err != nil {
723+
return nil, xerrors.Errorf("Failed to convert libs. err: %w", err)
780724
}
725+
libraryScanners = append(libraryScanners, libscan...)
781726
return libraryScanners, nil
782727
}
783728

729+
// https://github.com/aquasecurity/trivy/blob/84677903a6fa1b707a32d0e8b2bffc23dde52afa/pkg/fanal/analyzer/const.go
730+
var disabledAnalyzers = []analyzer.Type{
731+
// ======
732+
// OS
733+
// ======
734+
analyzer.TypeOSRelease,
735+
analyzer.TypeAlpine,
736+
analyzer.TypeAmazon,
737+
analyzer.TypeCBLMariner,
738+
analyzer.TypeDebian,
739+
analyzer.TypePhoton,
740+
analyzer.TypeCentOS,
741+
analyzer.TypeRocky,
742+
analyzer.TypeAlma,
743+
analyzer.TypeFedora,
744+
analyzer.TypeOracle,
745+
analyzer.TypeRedHatBase,
746+
analyzer.TypeSUSE,
747+
analyzer.TypeUbuntu,
748+
749+
// OS Package
750+
analyzer.TypeApk,
751+
analyzer.TypeDpkg,
752+
analyzer.TypeDpkgLicense,
753+
analyzer.TypeRpm,
754+
analyzer.TypeRpmqa,
755+
756+
// OS Package Repository
757+
analyzer.TypeApkRepo,
758+
759+
// ============
760+
// Image Config
761+
// ============
762+
analyzer.TypeApkCommand,
763+
764+
// =================
765+
// Structured Config
766+
// =================
767+
analyzer.TypeYaml,
768+
analyzer.TypeJSON,
769+
analyzer.TypeDockerfile,
770+
analyzer.TypeTerraform,
771+
analyzer.TypeCloudFormation,
772+
analyzer.TypeHelm,
773+
774+
// ========
775+
// License
776+
// ========
777+
analyzer.TypeLicenseFile,
778+
779+
// ========
780+
// Secrets
781+
// ========
782+
analyzer.TypeSecret,
783+
784+
// =======
785+
// Red Hat
786+
// =======
787+
analyzer.TypeRedHatContentManifestType,
788+
analyzer.TypeRedHatDockerfileType,
789+
}
790+
784791
// DummyFileInfo is a dummy struct for libscan
785792
type DummyFileInfo struct {
786793
size int64

‎scanner/debian.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1155,7 +1155,7 @@ func (o *debian) checkrestart() error {
11551155
o.Packages[p.Name] = pack
11561156

11571157
for j, proc := range p.NeedRestartProcs {
1158-
if proc.HasInit == false {
1158+
if !proc.HasInit {
11591159
continue
11601160
}
11611161
packs[i].NeedRestartProcs[j].InitSystem = initName

‎scanner/executil.go

-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,6 @@ func parallelExec(fn func(osTypeInterface) error, timeoutSec ...int) {
128128
}
129129
}
130130
servers = successes
131-
return
132131
}
133132

134133
func exec(c config.ServerInfo, cmd string, sudo bool, log ...logging.Logger) (result execResult) {

‎subcmds/discover.go

+1
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@ host = "{{$ip}}"
216216
#type = "pseudo"
217217
#memo = "DB Server"
218218
#findLock = true
219+
#findLockDirs = [ "/path/to/prject/lib" ]
219220
#lockfiles = ["/path/to/package-lock.json"]
220221
#cpeNames = [ "cpe:/a:rubyonrails:ruby_on_rails:4.2.1" ]
221222
#owaspDCXMLPath = "/path/to/dependency-check-report.xml"

0 commit comments

Comments
 (0)