Merge pull request #5870 from projectdiscovery/dev

v3.3.7

Author: ehsandeep

.gitignore

@@ -2,32 +2,32 @@
 **/*-cache
 **/*-config
 **/.cache
-*.DS_Store
-*.exe
+**/*.DS_Store
+**/*.exe
 .devcontainer
 .gitignore
 .idea
 .vscode
 
 # Binaries
 /bin/*
-**/bindgen
-**/debug-*
-**/docgen
-**/functional-test
-**/fuzzplayground
-**/integration-test
-**/jsdocgen
-**/main
-**/memogen
-**/nuclei
-**/nuclei-stats*
-**/nuclei_dev
-**/nuclei_main
-**/scan-charts
-**/scrapefunc
-**/scrapefuncs
-**/tsgen
+/bindgen
+/debug-*
+/docgen
+/functional-test
+/fuzzplayground
+/integration-test
+/jsdocgen
+/main
+/memogen
+/nuclei
+/nuclei-stats*
+/nuclei_dev
+/nuclei_main
+/scan-charts
+/scrapefunc
+/scrapefuncs
+/tsgen
 
 # Templates
 /*.yaml

Makefile

@@ -42,6 +42,10 @@ scan-charts: GOBUILD_OUTPUT = ./bin/scan-charts
 scan-charts: GOBUILD_PACKAGES = cmd/scan-charts/main.go
 scan-charts: go-build
 
+template-signer: GOBUILD_OUTPUT = ./bin/template-signer
+template-signer: GOBUILD_PACKAGES = cmd/tools/signer/main.go
+template-signer: go-build
+
 docgen: GOBUILD_OUTPUT = ./bin/docgen
 docgen: GOBUILD_PACKAGES = cmd/docgen/docgen.go
 docgen: bin = dstdocgen

README.md

@@ -264,6 +264,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),
 		flagSet.BoolVarP(&options.DisableUnsignedTemplates, "disable-unsigned-templates", "dut", false, "disable running unsigned templates or templates with mismatched signature"),
 		flagSet.BoolVarP(&options.EnableSelfContainedTemplates, "enable-self-contained", "esc", false, "enable loading self-contained templates"),
+		flagSet.BoolVarP(&options.EnableGlobalMatchersTemplates, "enable-global-matchers", "egm", false, "enable loading global matchers templates"),
 		flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
 	)
 

cmd/nuclei/main.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"flag"
+	"os"
+	"path/filepath"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	fileutil "github.com/projectdiscovery/utils/file"
+	folderutil "github.com/projectdiscovery/utils/folder"
+)
+
+var (
+	appConfigDir    = folderutil.AppConfigDirOrDefault(".config", "nuclei")
+	defaultCertFile = filepath.Join(appConfigDir, "keys", "nuclei-user.crt")
+	defaultPrivKey  = filepath.Join(appConfigDir, "keys", "nuclei-user-private-key.pem")
+)
+
+var (
+	template string
+	cert     string
+	privKey  string
+)
+
+func main() {
+	flag.StringVar(&template, "template", "", "template to sign (file only)")
+	flag.StringVar(&cert, "cert", defaultCertFile, "certificate file")
+	flag.StringVar(&privKey, "priv-key", defaultPrivKey, "private key file")
+	flag.Parse()
+
+	config.DefaultConfig.LogAllEvents = true
+	gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
+
+	if template == "" {
+		gologger.Fatal().Msg("template is required")
+	}
+	if !fileutil.FileExists(template) {
+		gologger.Fatal().Msgf("template file %s does not exist or not a file", template)
+	}
+
+	// get signer
+	tmplSigner, err := signer.NewTemplateSignerFromFiles(cert, privKey)
+	if err != nil {
+		gologger.Fatal().Msgf("failed to create signer: %s", err)
+	}
+	gologger.Info().Msgf("Template Signer: %v\n", tmplSigner.Identifier())
+
+	// read file
+	bin, err := os.ReadFile(template)
+	if err != nil {
+		gologger.Fatal().Msgf("failed to read template file %s: %s", template, err)
+	}
+
+	// extract signature and content
+	sig, content := signer.ExtractSignatureAndContent(bin)
+	hash := sha256.Sum256(content)
+
+	gologger.Info().Msgf("Signature Details:")
+	gologger.Info().Msgf("----------------")
+	gologger.Info().Msgf("Signature: %s", sig)
+	gologger.Info().Msgf("Content Hash (SHA256): %s\n", hex.EncodeToString(hash[:]))
+
+	execOpts := defaultExecutorOpts(template)
+
+	tmpl, err := templates.Parse(template, nil, execOpts)
+	if err != nil {
+		gologger.Fatal().Msgf("failed to parse template: %s", err)
+	}
+	gologger.Info().Msgf("Template Verified: %v\n", tmpl.Verified)
+
+	if !tmpl.Verified {
+		gologger.Info().Msgf("------------------------")
+		gologger.Info().Msg("Template is not verified, signing template")
+		if err := templates.SignTemplate(tmplSigner, template); err != nil {
+			gologger.Fatal().Msgf("Failed to sign template: %s", err)
+		}
+		// verify again by reading file what the new signature and hash is
+		bin2, err := os.ReadFile(template)
+		if err != nil {
+			gologger.Fatal().Msgf("failed to read signed template file %s: %s", template, err)
+		}
+		sig2, content2 := signer.ExtractSignatureAndContent(bin2)
+		hash2 := sha256.Sum256(content2)
+
+		gologger.Info().Msgf("Updated Signature Details:")
+		gologger.Info().Msgf("------------------------")
+		gologger.Info().Msgf("Signature: %s", sig2)
+		gologger.Info().Msgf("Content Hash (SHA256): %s\n", hex.EncodeToString(hash2[:]))
+	}
+	gologger.Info().Msgf("✓ Template signed & verified successfully")
+}
+
+func defaultExecutorOpts(templatePath string) protocols.ExecutorOptions {
+	// use parsed options when initializing signer instead of default options
+	options := types.DefaultOptions()
+	templates.UseOptionsForSigner(options)
+	catalog := disk.NewCatalog(filepath.Dir(templatePath))
+	executerOpts := protocols.ExecutorOptions{
+		Catalog:      catalog,
+		Options:      options,
+		TemplatePath: templatePath,
+		Parser:       templates.NewParser(),
+	}
+	return executerOpts
+}

cmd/tools/signer/main.go

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.1.1
-	github.com/projectdiscovery/fastdialer v0.2.9
-	github.com/projectdiscovery/hmap v0.0.67
+	github.com/projectdiscovery/fastdialer v0.2.10
+	github.com/projectdiscovery/hmap v0.0.69
 	github.com/projectdiscovery/interactsh v1.2.0
-	github.com/projectdiscovery/rawhttp v0.1.74
-	github.com/projectdiscovery/retryabledns v1.0.85
-	github.com/projectdiscovery/retryablehttp-go v1.0.86
+	github.com/projectdiscovery/rawhttp v0.1.76
+	github.com/projectdiscovery/retryabledns v1.0.86
+	github.com/projectdiscovery/retryablehttp-go v1.0.88
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -38,9 +38,9 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2
 	github.com/xanzy/go-gitlab v0.107.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.30.0
+	golang.org/x/net v0.31.0
 	golang.org/x/oauth2 v0.22.0
-	golang.org/x/text v0.19.0
+	golang.org/x/text v0.20.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -85,29 +85,29 @@ require (
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
 	github.com/projectdiscovery/goflags v0.1.65
-	github.com/projectdiscovery/gologger v1.1.31
+	github.com/projectdiscovery/gologger v1.1.33
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.3
 	github.com/projectdiscovery/httpx v1.6.9
 	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.61
+	github.com/projectdiscovery/ratelimit v0.0.64
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.8
 	github.com/projectdiscovery/uncover v1.0.9
 	github.com/projectdiscovery/useragent v0.0.78
-	github.com/projectdiscovery/utils v0.2.18
-	github.com/projectdiscovery/wappalyzergo v0.2.2
+	github.com/projectdiscovery/utils v0.3.0
+	github.com/projectdiscovery/wappalyzergo v0.2.5
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
 	github.com/yassinebenaid/godump v0.10.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
 	go.mongodb.org/mongo-driver v1.17.0
-	golang.org/x/term v0.25.0
+	golang.org/x/term v0.26.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -152,6 +152,7 @@ require (
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
+	github.com/felixge/fgprof v0.9.5 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gaissmai/bart v0.9.5 // indirect
@@ -246,7 +247,7 @@ require (
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sync v0.9.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	mellium.im/sasl v0.3.1 // indirect
 )
@@ -313,10 +314,10 @@ require (
 	go.etcd.io/bbolt v1.3.10 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
 	google.golang.org/protobuf v1.34.2 // indirect