Skip to content

Commit 0f493e2

Browse files
x51pwnx51pwn
committed
优化指纹算法；增加工作流程图
1 parent 0dd0fca commit 0f493e2

File tree

9 files changed

+115
-110
lines changed

9 files changed

+115
-110
lines changed

‎.github/up.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
cat ./go.mod|grep projectdiscovery|grep -E "subfinder|nuclei"|awk '{print $1}'|xargs -I % go get -u %
1+
cat ./go.mod|grep projectdiscovery|grep -E "subfinder|nuclei|wappalyzergo"|awk '{print $1}'|xargs -I % go get -u %
22

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
- 允许通过config/config.json配置定义自己的字典，或者设置相关的开关，可以在这里定义nuclei、httx、naabu的几个Options
3434

3535
# 工作流程
36+
<img src="static/workflow.jpg">
3637
- 0.【智能子域名爆破】集成Subfinder,当通过 export EnableSubfinder=true 开启后，ssl证书中的域名信息包含"*."开头时自动启动子域名遍历
3738
- 1.【端口扫描】集成Nuclei官方产品naabu ( > 2.1k)
3839
- 2.【服务识别】naabu调用系统安装的nmap，请先自行安装nmap

‎config/nuclei_esConfig.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
elasticsearch:
2+
# IP for elasticsearch instance
3+
ip: 127.0.0.1
4+
# Port is the port of elasticsearch instance
5+
port: 9200
6+
# IndexName is the name of the elasticsearch index
7+
index-name: nuclei_index
8+
# SSL enables ssl for elasticsearch connection
9+
ssl: false
10+
# SSLVerification disables SSL verification for elasticsearch
11+
ssl-verification: false
12+
# Username for the elasticsearch instance
13+
username: elastic
14+
# Password is the password for elasticsearch instance
15+
password: test

‎nuclei_Yaml/nuclei_yaml.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ func RunNuclei(buf bytes.Buffer, xx chan bool) {
4444
}
4545
}
4646
options.Verbose = false
47+
options.Stream = false
4748
////////////////////////////////////*/
4849

4950
nucleiRunner, err := runner.New(options)
@@ -224,7 +225,7 @@ func readConfig() {
224225
options.Retries = 1
225226
options.LeaveDefaultPorts = false
226227
options.MaxHostError = 30
227-
options.Project = false
228+
options.Project = true // 去重复
228229
options.ProjectPath = os.TempDir()
229230
options.StopAtFirstMatch = false
230231
options.Stream = false

‎pkg/fingerprint/fingerScan.go

Lines changed: 40 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -28,93 +28,59 @@ func mapToJson(param map[string][]string) string {
2828
return dataString
2929
}
3030

31+
// 合并所有指纹需要请求的链接，也就是合并所有请求，相同的只请求一次
32+
// 会多次调用，所以需要cache中间结果
33+
func PreprocessingFingerScan(url string) []string {
34+
// 有时间再实现
35+
return []string{}
36+
}
37+
3138
func FingerScan(headers map[string][]string, body []byte, title string, url string) []string {
3239
bodyString := string(body)
3340
headersjson := mapToJson(headers)
3441
favhash := getfavicon(bodyString, url)
3542
var cms []string
36-
for _, finp := range EholeFinpx.Fingerprint {
37-
if finp.Location == "body" {
38-
if finp.Method == "keyword" {
39-
if iskeyword(bodyString, finp.Keyword) {
40-
cms = append(cms, finp.Cms)
41-
}
42-
}
43-
if finp.Method == "faviconhash" {
44-
if favhash == finp.Keyword[0] {
45-
cms = append(cms, finp.Cms)
46-
}
47-
}
48-
if finp.Method == "regular" {
49-
if isregular(bodyString, finp.Keyword) {
50-
cms = append(cms, finp.Cms)
51-
}
52-
}
53-
}
54-
if finp.Location == "header" {
55-
if finp.Method == "keyword" {
56-
if iskeyword(headersjson, finp.Keyword) {
57-
cms = append(cms, finp.Cms)
58-
}
59-
}
60-
if finp.Method == "regular" {
61-
if isregular(headersjson, finp.Keyword) {
62-
cms = append(cms, finp.Cms)
63-
}
64-
}
65-
}
66-
if finp.Location == "title" {
67-
if finp.Method == "keyword" {
68-
if iskeyword(title, finp.Keyword) {
69-
cms = append(cms, finp.Cms)
43+
for _, x1 := range []*Packjson{EholeFinpx, LocalFinpx} {
44+
for _, finp := range x1.Fingerprint {
45+
if finp.Location == "body" {
46+
if finp.Method == "keyword" {
47+
if iskeyword(bodyString, finp.Keyword) {
48+
cms = append(cms, finp.Cms)
49+
}
7050
}
71-
}
72-
if finp.Method == "regular" {
73-
if isregular(title, finp.Keyword) {
74-
cms = append(cms, finp.Cms)
51+
if finp.Method == "faviconhash" {
52+
if favhash == finp.Keyword[0] {
53+
cms = append(cms, finp.Cms)
54+
}
7555
}
76-
}
77-
}
78-
}
79-
for _, finp := range LocalFinpx.Fingerprint {
80-
if finp.Location == "body" {
81-
if finp.Method == "keyword" {
82-
if iskeyword(bodyString, finp.Keyword) {
83-
cms = append(cms, finp.Cms)
56+
if finp.Method == "regular" {
57+
if isregular(bodyString, finp.Keyword) {
58+
cms = append(cms, finp.Cms)
59+
}
8460
}
8561
}
86-
if finp.Method == "faviconhash" {
87-
if favhash == finp.Keyword[0] {
88-
cms = append(cms, finp.Cms)
62+
if finp.Location == "header" {
63+
if finp.Method == "keyword" {
64+
if iskeyword(headersjson, finp.Keyword) {
65+
cms = append(cms, finp.Cms)
66+
}
8967
}
90-
}
91-
if finp.Method == "regular" {
92-
if isregular(bodyString, finp.Keyword) {
93-
cms = append(cms, finp.Cms)
68+
if finp.Method == "regular" {
69+
if isregular(headersjson, finp.Keyword) {
70+
cms = append(cms, finp.Cms)
71+
}
9472
}
9573
}
96-
}
97-
if finp.Location == "header" {
98-
if finp.Method == "keyword" {
99-
if iskeyword(headersjson, finp.Keyword) {
100-
cms = append(cms, finp.Cms)
74+
if finp.Location == "title" {
75+
if finp.Method == "keyword" {
76+
if iskeyword(title, finp.Keyword) {
77+
cms = append(cms, finp.Cms)
78+
}
10179
}
102-
}
103-
if finp.Method == "regular" {
104-
if isregular(headersjson, finp.Keyword) {
105-
cms = append(cms, finp.Cms)
106-
}
107-
}
108-
}
109-
if finp.Location == "title" {
110-
if finp.Method == "keyword" {
111-
if iskeyword(title, finp.Keyword) {
112-
cms = append(cms, finp.Cms)
113-
}
114-
}
115-
if finp.Method == "regular" {
116-
if isregular(title, finp.Keyword) {
117-
cms = append(cms, finp.Cms)
80+
if finp.Method == "regular" {
81+
if isregular(title, finp.Keyword) {
82+
cms = append(cms, finp.Cms)
83+
}
11884
}
11985
}
12086
}

‎pkg/httpx/runner/runner.go

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,21 @@ import (
2929
"time"
3030

3131
"github.com/bluele/gcache"
32+
"github.com/hktalent/scan4all/pkg/httpx/common/hashes"
3233
"github.com/logrusorgru/aurora"
3334
"github.com/pkg/errors"
3435
"github.com/projectdiscovery/clistats"
3536
"github.com/projectdiscovery/cryptoutil"
3637
"github.com/projectdiscovery/goconfig"
3738
"github.com/projectdiscovery/stringsutil"
3839
"github.com/projectdiscovery/urlutil"
39-
"github.com/hktalent/scan4all/pkg/httpx/common/hashes"
4040

41+
customport "github.com/hktalent/scan4all/pkg/httpx/common/customports"
42+
fileutilz "github.com/hktalent/scan4all/pkg/httpx/common/fileutil"
43+
"github.com/hktalent/scan4all/pkg/httpx/common/httputilz"
44+
"github.com/hktalent/scan4all/pkg/httpx/common/httpx"
45+
"github.com/hktalent/scan4all/pkg/httpx/common/slice"
46+
"github.com/hktalent/scan4all/pkg/httpx/common/stringz"
4147
// automatic fd max increase if running as root
4248
_ "github.com/projectdiscovery/fdmax/autofdmax"
4349
"github.com/projectdiscovery/fileutil"
@@ -50,12 +56,6 @@ import (
5056
"github.com/projectdiscovery/retryablehttp-go"
5157
wappalyzer "github.com/projectdiscovery/wappalyzergo"
5258
"github.com/remeh/sizedwaitgroup"
53-
customport "github.com/hktalent/scan4all/pkg/httpx/common/customports"
54-
fileutilz "github.com/hktalent/scan4all/pkg/httpx/common/fileutil"
55-
"github.com/hktalent/scan4all/pkg/httpx/common/httputilz"
56-
"github.com/hktalent/scan4all/pkg/httpx/common/httpx"
57-
"github.com/hktalent/scan4all/pkg/httpx/common/slice"
58-
"github.com/hktalent/scan4all/pkg/httpx/common/stringz"
5959
"go.uber.org/ratelimit"
6060
)
6161

@@ -700,12 +700,20 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
700700
continue
701701
}
702702
r.process(tt, wg, hp, protocol, scanopts, output)
703+
a1 := fingerprint.PreprocessingFingerScan(tt)
704+
for _, x1 := range a1 {
705+
r.process(x1, wg, hp, protocol, scanopts, output)
706+
}
703707
}
704708
for _, tt := range result.TLSData.CommonName {
705709
if !r.testAndSet(tt) {
706710
continue
707711
}
708712
r.process(tt, wg, hp, protocol, scanopts, output)
713+
a1 := fingerprint.PreprocessingFingerScan(tt)
714+
for _, x1 := range a1 {
715+
r.process(x1, wg, hp, protocol, scanopts, output)
716+
}
709717
}
710718
}
711719
if scanopts.CSPProbe && result.CSPData != nil {
@@ -715,6 +723,10 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
715723
continue
716724
}
717725
r.process(tt, wg, hp, protocol, scanopts, output)
726+
a1 := fingerprint.PreprocessingFingerScan(tt)
727+
for _, x1 := range a1 {
728+
r.process(x1, wg, hp, protocol, scanopts, output)
729+
}
718730
}
719731
}
720732
}(target, method, prot)

‎pkg/naabu/v2/pkg/runner/runner.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ func (r *Runner) Httpxrun() error {
5353
var nucleiDone = make(chan bool)
5454
// 集成nuclei
5555
//log.Println("httpxrunner.Naabubuffer = ", httpxrunner.Naabubuffer.String())
56+
//Naabubuffer1 := bytes.Buffer{}
57+
//Naabubuffer1.Write(httpxrunner.Naabubuffer.Bytes())
5658
go nuclei_Yaml.RunNuclei(httpxrunner.Naabubuffer, nucleiDone)
5759
httpxoptions := httpxrunner.ParseOptions()
5860
httpxoptions.Output = r.options.Output

‎static/workflow.jpg

570 KB
Loading

‎test/testXml.go

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,44 @@
11
package main
22

33
import (
4-
"github.com/hktalent/scan4all/pkg/hydra"
5-
"io/ioutil"
4+
"bytes"
5+
"github.com/hktalent/scan4all/nuclei_Yaml"
66
)
77

8+
var Naabubuffer bytes.Buffer = bytes.Buffer{}
9+
810
func main() {
11+
var nucleiDone = make(chan bool)
12+
Naabubuffer.Write([]byte("192.168.10.31\n"))
13+
// 集成nuclei
14+
//log.Println("httpxrunner.Naabubuffer = ", httpxrunner.Naabubuffer.String())
15+
nuclei_Yaml.RunNuclei(Naabubuffer, nucleiDone)
16+
<-nucleiDone
917

10-
x := "test/4ee58a18fc884edd74ff1ec077e8c90c6048a45b.xml"
11-
b, err := ioutil.ReadFile(x)
12-
if nil == err && 0 < len(b) {
13-
s := string(b)
14-
hydra.DoParseXml(s)
15-
//select {}
16-
//doc, err := xmlquery.Parse(strings.NewReader(s))
17-
//if err != nil {
18-
// log.Println(err)
19-
// return
20-
//}
21-
//
22-
//for _, n := range xmlquery.Find(doc, "//host") {
23-
// x1 := n.SelectElement("address").Attr[0].Value
24-
// ps := n.SelectElements("ports/port")
25-
// for _, x := range ps {
26-
// if "open" == x.SelectElement("state").Attr[0].Value {
27-
// ip := x1
28-
// port, _ := strconv.Atoi(GetAttr(x.Attr, "portid"))
29-
// service := GetAttr(x.SelectElement("service").Attr, "name")
30-
// fmt.Printf("%s\t%d\t%s\n", ip, port, service)
31-
// }
32-
// }
33-
//}
34-
}
18+
//x := "test/4ee58a18fc884edd74ff1ec077e8c90c6048a45b.xml"
19+
//b, err := ioutil.ReadFile(x)
20+
//if nil == err && 0 < len(b) {
21+
// s := string(b)
22+
// hydra.DoParseXml(s)
23+
// //select {}
24+
// //doc, err := xmlquery.Parse(strings.NewReader(s))
25+
// //if err != nil {
26+
// // log.Println(err)
27+
// // return
28+
// //}
29+
// //
30+
// //for _, n := range xmlquery.Find(doc, "//host") {
31+
// // x1 := n.SelectElement("address").Attr[0].Value
32+
// // ps := n.SelectElements("ports/port")
33+
// // for _, x := range ps {
34+
// // if "open" == x.SelectElement("state").Attr[0].Value {
35+
// // ip := x1
36+
// // port, _ := strconv.Atoi(GetAttr(x.Attr, "portid"))
37+
// // service := GetAttr(x.SelectElement("service").Attr, "name")
38+
// // fmt.Printf("%s\t%d\t%s\n", ip, port, service)
39+
// // }
40+
// // }
41+
// //}
42+
//}
3543

3644
}

0 commit comments

Comments
 (0)