Skip to content

Use fs.FS when explicitly given #5312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 22, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion lib/sdk_private.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,10 @@ func (e *NucleiEngine) init(ctx context.Context) error {
// and also upgrade templates to latest version if available
installer.NucleiSDKVersionCheck()

return e.processUpdateCheckResults()
if DefaultConfig.CanCheckForUpdates() {
return e.processUpdateCheckResults()
}
return nil
}

type syncOnce struct {
Expand Down
35 changes: 25 additions & 10 deletions pkg/catalog/disk/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,35 +5,50 @@ import (
"io/fs"
"os"

"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
)

// DiskCatalog is a template catalog helper implementation based on disk
type DiskCatalog struct {
templatesDirectory string
templatesFS fs.FS // TODO: Refactor to use this
templatesFS fs.FS // Due to issues with how Go has implemented fs.FS, we'll have to also implement normal os operations, as well. See: https://github.com/golang/go/issues/44279
}

// NewCatalog creates a new Catalog structure using provided input items
// using disk based items
func NewCatalog(directory string) *DiskCatalog {
catalog := &DiskCatalog{templatesDirectory: directory}
if directory != "" {
catalog.templatesFS = os.DirFS(directory)
} else {
catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir())
if directory == "" {
catalog.templatesDirectory = config.DefaultConfig.GetTemplateDir()
}
return catalog
}

// NewFSCatalog creates a new Catalog structure using provided input items
// using the fs.FS as its filesystem.
func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog {
catalog := &DiskCatalog{
templatesDirectory: directory,
templatesFS: fs,
}
return catalog
}

// OpenFile opens a file and returns an io.ReadCloser to the file.
// It is used to read template and payload files based on catalog responses.
func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
file, err := os.Open(filename)
if err != nil {
if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
return file, nil
gologger.Debug().Msgf("DiskCatalog: OpenFile: %s", filename)

if d.templatesFS == nil {
file, err := os.Open(filename)
if err != nil {
if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
return file, nil
}
}
return file, err
}
return file, err

return d.templatesFS.Open(filename)
}
157 changes: 120 additions & 37 deletions pkg/catalog/disk/find.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ var deprecatedPathsCounter int

// GetTemplatesPath returns a list of absolute paths for the provided template list.
func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[string]error) {
gologger.Debug().Msgf("DiskCatalog: GetTemplatesPath: %q", definitions)

// keeps track of processed dirs and files
processed := make(map[string]bool)
allTemplates := []string{}
Expand Down Expand Up @@ -65,6 +67,8 @@ func (c *DiskCatalog) GetTemplatesPath(definitions []string) ([]string, map[stri
// list of finished absolute paths to the templates evaluating any glob patterns
// or folders provided as in.
func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
gologger.Debug().Msgf("DiskCatalog: GetTemplatePath: %q", target)

processed := make(map[string]struct{})
// Template input includes a wildcard
if strings.Contains(target, "*") {
Expand All @@ -79,17 +83,21 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
}

// try to handle deprecated template paths
absPath := BackwardsCompatiblePaths(c.templatesDirectory, target)
if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
if config.DefaultConfig.LogAllEvents {
gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
absPath := target
if c.templatesFS == nil {
absPath = BackwardsCompatiblePaths(c.templatesDirectory, target)
if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
if config.DefaultConfig.LogAllEvents {
gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
}
deprecatedPathsCounter++
}
deprecatedPathsCounter++
}

absPath, err := c.convertPathToAbsolute(absPath)
if err != nil {
return nil, errors.Wrapf(err, "could not find template file")
var err error
absPath, err = c.convertPathToAbsolute(absPath)
if err != nil {
return nil, errors.Wrapf(err, "could not find template file")
}
}

// Template input is either a file or a directory
Expand Down Expand Up @@ -132,6 +140,8 @@ func (c *DiskCatalog) convertPathToAbsolute(t string) (string, error) {

// findGlobPathMatches returns the matched files from a glob path
func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) {
gologger.Debug().Msgf("DiskCatalog: findGlobPathMatches: %q", absPath)

// to support globbing on old paths we use brute force to find matches with exit on first match
// trim templateDir if any
relPath := strings.TrimPrefix(absPath, c.templatesDirectory)
Expand All @@ -143,24 +153,60 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
if c.templatesDirectory == "" {
templateDir = "./"
}
matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
if len(matches) != 0 {

if c.templatesFS == nil {
matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
if len(matches) != 0 {
return matches
}

// condition to support network cve related globs
matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
return matches
} else {
sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http"))
if err != nil {
return nil
}
matches, _ := fs.Glob(sub, inputGlob)
if len(matches) != 0 {
return matches
}

// condition to support network cve related globs
sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network"))
if err != nil {
return nil
}
matches, _ = fs.Glob(sub, inputGlob)
return matches
}
// condition to support network cve related globs
matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
return matches
}

var matched []string
matches, err := fs.Glob(c.templatesFS, relPath)
if len(matches) != 0 {
matched = append(matched, matches...)
var matches []string
if c.templatesFS == nil {
var err error
matches, err = filepath.Glob(relPath)
if len(matches) != 0 {
matched = append(matched, matches...)
} else {
matched = append(matched, OldPathsResolver(relPath)...)
}
if err != nil && len(matched) == 0 {
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
} else {
matched = append(matched, OldPathsResolver(relPath)...)
}
if err != nil && len(matched) == 0 {
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
var err error
matches, err = fs.Glob(c.templatesFS, relPath)
if len(matches) != 0 {
matched = append(matched, matches...)
} else {
matched = append(matched, OldPathsResolver(relPath)...)
}
if err != nil && len(matched) == 0 {
return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
}
results := make([]string, 0, len(matches))
for _, match := range matches {
Expand All @@ -175,11 +221,25 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
info, err := os.Stat(absPath)
if c.templatesFS != nil {
absPath = strings.TrimPrefix(absPath, "/")
}
gologger.Debug().Msgf("DiskCatalog: findFileMatches: %q (valid: %t)", absPath, fs.ValidPath(absPath))

var info fs.File
if c.templatesFS == nil {
info, err = os.Open(absPath)
} else {
info, err = c.templatesFS.Open(absPath)
}
if err != nil {
return "", false, err
}
stat, err := info.Stat()
if err != nil {
return "", false, err
}
if !info.Mode().IsRegular() {
if !stat.Mode().IsRegular() {
return "", false, nil
}
if _, ok := processed[absPath]; !ok {
Expand All @@ -191,23 +251,46 @@ func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struc

// findDirectoryMatches finds matches for templates from a directory
func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
gologger.Debug().Msgf("DiskCatalog: findDirectoryMatches: %q", absPath)

var results []string
err := filepath.WalkDir(
absPath,
func(path string, d fs.DirEntry, err error) error {
// continue on errors
if err != nil {
var err error
if c.templatesFS == nil {
err = filepath.WalkDir(
absPath,
func(path string, d fs.DirEntry, err error) error {
// continue on errors
if err != nil {
return nil
}
if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
if _, ok := processed[path]; !ok {
results = append(results, path)
processed[path] = struct{}{}
}
}
return nil
}
if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
if _, ok := processed[path]; !ok {
results = append(results, path)
processed[path] = struct{}{}
},
)
} else {
err = fs.WalkDir(
c.templatesFS,
absPath,
func(path string, d fs.DirEntry, err error) error {
// continue on errors
if err != nil {
return nil
}
}
return nil
},
)
if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
if _, ok := processed[path]; !ok {
results = append(results, path)
processed[path] = struct{}{}
}
}
return nil
},
)
}
return results, err
}

Expand Down
3 changes: 3 additions & 0 deletions pkg/catalog/disk/path.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"

"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
fileutil "github.com/projectdiscovery/utils/file"
urlutil "github.com/projectdiscovery/utils/url"
Expand All @@ -18,6 +19,8 @@ import (
// or checking the nuclei templates directory. If a second path is given,
// it also tries to find paths relative to that second path.
func (c *DiskCatalog) ResolvePath(templateName, second string) (string, error) {
gologger.Debug().Msgf("DiskCatalog: ResolvePath: %q, %q", templateName, second)

if filepath.IsAbs(templateName) {
return templateName, nil
}
Expand Down