Skip to content

Commit d42caac

Browse files
committed
重构被动模式,同时修正nuclei多实例bug 2022-10-05
1 parent 3f88829 commit d42caac

File tree

35 files changed

+854
-287
lines changed

35 files changed

+854
-287
lines changed

brute/filefuzz.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
_ "embed"
66
"github.com/antlabs/strsim"
7+
"github.com/hktalent/goSqlite_gorm/lib/scan/Const"
8+
"github.com/hktalent/goSqlite_gorm/pkg/models"
79
"github.com/hktalent/scan4all/lib/util"
810
"log"
911
"net/url"
@@ -139,6 +141,12 @@ func init() {
139141
// 随机10个字符串
140142
var RandStr4Cookie = util.RandStringRunes(10)
141143

144+
// 基于工厂方法构建
145+
var FileFuzz4Engin = util.EngineFuncFactory(func(evt *models.EventData, args ...interface{}) {
146+
filePaths, fileFuzzTechnologies := FileFuzz(evt.Task.ScanWeb, 200, 100, "")
147+
util.SendEngineLog(evt, Const.ScanType_WebDirScan, filePaths, fileFuzzTechnologies)
148+
})
149+
142150
// 重写了fuzz:优化流程、优化算法、修复线程安全bug、增加智能功能
143151
// 两次 ioutil.ReadAll(resp.Body),第二次就会 Read返回EOF error
144152
func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
@@ -149,7 +157,6 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
149157
if eableFileFuzz || util.TestRepeat(u, "FileFuzz") {
150158
return []string{}, []string{}
151159
}
152-
153160
//log.Println("start file fuzz", u)
154161
var (
155162
//path404 = RandStr // 绝对404页面路径

engine/dispather.go

Lines changed: 35 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,33 @@ package engine
22

33
import (
44
"github.com/hktalent/goSqlite_gorm/lib"
5-
"github.com/hktalent/goSqlite_gorm/lib/scan/Const"
5+
. "github.com/hktalent/goSqlite_gorm/lib/scan/Const"
66
"github.com/hktalent/goSqlite_gorm/pkg/models"
7+
"github.com/hktalent/scan4all/brute"
78
"github.com/hktalent/scan4all/lib/util"
89
"github.com/hktalent/scan4all/pkg/portScan"
10+
"github.com/hktalent/scan4all/pocs_go"
11+
"github.com/hktalent/scan4all/projectdiscovery/nuclei_Yaml"
912
)
1013

14+
// passive 被动模式
1115
var (
1216
CaseScanFunc = map[int]util.EngineFuncType{
13-
Const.ScanType_SSLInfo: nil, // 01- SSL信息分析,并对域名信息进行收集、进入下一步流程
14-
Const.ScanType_SubDomain: nil, // 02- 子域名爆破,新域名回归 到: 1 <-- -> 2,做去重处理
15-
Const.ScanType_MergeIps: nil, // 03- 默认自动合并ip,记录ip与域名的关联关系,再发送payload时考虑:相同ip不同域名,相同payload分别发送 合并相同目标 若干域名的ip,避免扫描时重复
16-
Const.ScanType_Pswd4hydra: nil, // 04- 密码破解,隐含包含了: 端口扫描(05-masscan + 06-nmap)
17-
Const.ScanType_Masscan: portScan.MassScanTarget, // 05- 合并后的ip 进行快速端口扫描
18-
Const.ScanType_Nmap: portScan.DoNmap, // 06、精准 端口指纹,排除masscan已经识别的几种指纹
19-
Const.ScanType_IpInfo: nil, // 07- 获取ip info
20-
Const.ScanType_GoPoc: nil, // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
21-
Const.ScanType_PortsWeb: nil, // 09- web端口识别,Naabu,识别 https,识别存活的web端口,再进入下一流程
22-
Const.ScanType_WebFingerprints: nil, // 10- web指纹,识别蜜罐,并标识
23-
Const.ScanType_WebDetectWaf: nil, // 11- detect WAF
24-
Const.ScanType_WebScrapy: nil, // 12- 爬虫分析,form表单识别,字段名识别,form action提取;
25-
Const.ScanType_WebInfo: nil, // 13- server、x-powerby、x***,url、ip、其他敏感信息(姓名、电话、地址、身份证)
26-
Const.ScanType_WebVulsScan: nil, // 14-nuclei
27-
Const.ScanType_WebDirScan: nil, // 14-dir爆破,Gobuster
17+
ScanType_SSLInfo: nil, // 01- SSL信息分析,并对域名信息进行收集、进入下一步流程
18+
ScanType_SubDomain: nil, // 02- 子域名爆破,新域名回归 到: 1 <-- -> 2,做去重处理
19+
ScanType_MergeIps: nil, // 03- 默认自动合并ip,记录ip与域名的关联关系,再发送payload时考虑:相同ip不同域名,相同payload分别发送 合并相同目标 若干域名的ip,避免扫描时重复
20+
ScanType_Pswd4hydra: nil, // 04- 密码破解,隐含包含了: 端口扫描(05-masscan + 06-nmap)
21+
ScanType_Masscan: portScan.MassScanTarget, // 05- 合并后的ip 进行快速端口扫描
22+
ScanType_Nmap: portScan.DoNmap, // 06、精准 端口指纹,排除masscan已经识别的几种指纹
23+
ScanType_IpInfo: nil, // 07- 获取ip info
24+
ScanType_GoPoc: pocs_go.POCcheck4Engin, // 08- go-poc 检测, 隐含包含了: ScanType_WebDirScan,端口扫描(05-masscan + 06-nmap)
25+
ScanType_PortsWeb: nil, // 09- web端口识别,Naabu,识别 https,识别存活的web端口,再进入下一流程
26+
ScanType_WebFingerprints: nil, // 10- web指纹,识别蜜罐,并标识
27+
ScanType_WebDetectWaf: nil, // 11- detect WAF
28+
ScanType_WebScrapy: nil, // 12- 爬虫分析,form表单识别,字段名识别,form action提取;
29+
ScanType_WebInfo: nil, // 13- server、x-powerby、x***,url、ip、其他敏感信息(姓名、电话、地址、身份证)
30+
ScanType_WebVulsScan: nuclei_Yaml.RunNucleiEngin, // 14-nuclei
31+
ScanType_WebDirScan: brute.FileFuzz4Engin, // 14-dir爆破,Gobuster,file fuzz
2832
}
2933
)
3034

@@ -35,37 +39,37 @@ func Dispather(task *models.Target4Chan) {
3539
if lib.HasScanType(task.ScanType, k) {
3640
x1 := &models.EventData{EventType: k, Task: task}
3741
switch k {
38-
case Const.ScanType_SSLInfo: // 01- SSL信息分析,并对域名信息进行收集、进入下一步流程
42+
case ScanType_SSLInfo: // 01- SSL信息分析,并对域名信息进行收集、进入下一步流程
3943
G_Engine.EventData <- x1
40-
case Const.ScanType_SubDomain: // 02- 子域名爆破,新域名回归 到: 1 <-- -> 2,做去重处理
44+
case ScanType_SubDomain: // 02- 子域名爆破,新域名回归 到: 1 <-- -> 2,做去重处理
4145
G_Engine.EventData <- x1
42-
case Const.ScanType_MergeIps: // 03- 默认自动合并ip,记录ip与域名的关联关系,再发送payload时考虑:相同ip不同域名,相同payload分别发送 合并相同目标 若干域名的ip,避免扫描时重复
46+
case ScanType_MergeIps: // 03- 默认自动合并ip,记录ip与域名的关联关系,再发送payload时考虑:相同ip不同域名,相同payload分别发送 合并相同目标 若干域名的ip,避免扫描时重复
4347
G_Engine.EventData <- x1
44-
case Const.ScanType_Pswd4hydra: // 04- 密码破解,隐含包含了: 端口扫描(05-masscan + 06-nmap)
48+
case ScanType_Pswd4hydra: // 04- 密码破解,隐含包含了: 端口扫描(05-masscan + 06-nmap)
4549
G_Engine.EventData <- x1
46-
case Const.ScanType_Masscan: // 05- 合并后的ip 进行快速端口扫描; // 06、精准 端口指纹,排除masscan已经识别的几种指纹
50+
case ScanType_Masscan: // 05- 合并后的ip 进行快速端口扫描; // 06、精准 端口指纹,排除masscan已经识别的几种指纹
4751
x1.EventData = []interface{}{[]interface{}{portScan.TargetStr(task.ScanWeb)}}
4852
G_Engine.EventData <- x1
49-
case Const.ScanType_Nmap: // 05- 合并后的ip 进行快速端口扫描; // 06、精准 端口指纹,排除masscan已经识别的几种指纹
53+
case ScanType_Nmap: // 05- 合并后的ip 进行快速端口扫描; // 06、精准 端口指纹,排除masscan已经识别的几种指纹
5054
x1.EventData = []interface{}{x1.Target2Ip(), []string{"0-65535"}}
5155
G_Engine.EventData <- x1
52-
case Const.ScanType_IpInfo: // 07- 获取ip info
56+
case ScanType_IpInfo: // 07- 获取ip info
5357
G_Engine.EventData <- x1
54-
case Const.ScanType_GoPoc: // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
58+
case ScanType_GoPoc: // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
5559
G_Engine.EventData <- x1
56-
case Const.ScanType_PortsWeb: // 09- web端口识别,Naabu,识别 https,识别存活的web端口,再进入下一流程
60+
case ScanType_PortsWeb: // 09- web端口识别,Naabu,识别 https,识别存活的web端口,再进入下一流程
5761
G_Engine.EventData <- x1
58-
case Const.ScanType_WebFingerprints: // 10- web指纹,识别蜜罐,并标识
62+
case ScanType_WebFingerprints: // 10- web指纹,识别蜜罐,并标识
5963
G_Engine.EventData <- x1
60-
case Const.ScanType_WebDetectWaf: // 11- detect WAF
64+
case ScanType_WebDetectWaf: // 11- detect WAF
6165
G_Engine.EventData <- x1
62-
case Const.ScanType_WebScrapy: // 12- 爬虫分析,form表单识别,字段名识别,form action提取;
66+
case ScanType_WebScrapy: // 12- 爬虫分析,form表单识别,字段名识别,form action提取;
6367
G_Engine.EventData <- x1
64-
case Const.ScanType_WebInfo: // 13- server、x-powerby、x***,url、ip、其他敏感信息(姓名、电话、地址、身份证)
68+
case ScanType_WebInfo: // 13- server、x-powerby、x***,url、ip、其他敏感信息(姓名、电话、地址、身份证)
6569
G_Engine.EventData <- x1
66-
case Const.ScanType_WebVulsScan: // 14-nuclei
70+
case ScanType_WebVulsScan: // 14-nuclei
6771
G_Engine.EventData <- x1
68-
case Const.ScanType_WebDirScan: // 14-dir爆破,Gobuster
72+
case ScanType_WebDirScan: // 14-dir爆破,Gobuster
6973
G_Engine.EventData <- x1
7074
default:
7175

engine/dispather_test.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package engine
2+
3+
import (
4+
"github.com/hktalent/goSqlite_gorm/lib/scan/Const"
5+
"github.com/hktalent/goSqlite_gorm/pkg/models"
6+
"github.com/hktalent/scan4all/lib/util"
7+
"os"
8+
"runtime"
9+
"testing"
10+
)
11+
12+
func TestDispather(t *testing.T) {
13+
os.Args = []string{"", "-host", "http://127.0.0.1", "-v"}
14+
runtime.GOMAXPROCS(runtime.NumCPU())
15+
util.DoInit(nil)
16+
Dispather(&models.Target4Chan{ScanWeb: "127.0.0.1", ScanType: Const.ScanType_Masscan})
17+
util.Wg.Wait()
18+
util.CloseAll()
19+
//for _, tt := range tests {
20+
// t.Run(tt.name, func(t *testing.T) {
21+
// Dispather(tt.args.task)
22+
// })
23+
//}
24+
}

engine/engine.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ func NewEngine(c *context.Context, pool int) *Engine {
3131
return G_Engine
3232
}
3333
G_Engine = &Engine{Context: c, Wg: util.Wg, Pool: pool, EventData: make(chan *models.EventData, pool)}
34-
3534
p, err := ants.NewPoolWithFunc(pool, func(i interface{}) {
3635
defer G_Engine.Wg.Done()
3736
G_Engine.DoEvent(i.(*models.EventData))

fixMod.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,6 @@ go build
1515
git add vendor
1616
rm -rf vendor/github.com/hktalent/goSqlite_gorm
1717
ln -s $HOME/MyWork/goSqlite_gorm $PWD/vendor/github.com/hktalent/goSqlite_gorm
18+
19+
gometalinter --install
20+
gometalinter --disable=gocyclo --disable=dupl --enable=goimports --disable=gas

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ require (
217217
github.com/hashicorp/go-version v1.6.0 // indirect
218218
github.com/hashicorp/golang-lru v0.5.4 // indirect
219219
github.com/hashicorp/hcl v1.0.0 // indirect
220-
github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea // indirect
220+
github.com/hktalent/go-utils v0.0.0-20221004095234-2e23f13b429d // indirect
221221
github.com/hktalent/websocket v0.0.0-20220908204337-b4a81b861976 // indirect
222222
github.com/iancoleman/orderedmap v0.2.0 // indirect
223223
github.com/itchyny/gojq v0.12.9 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,8 @@ github.com/hktalent/PipelineHttp v0.0.0-20221004080931-279351b9fb96 h1:8++Z/n334
510510
github.com/hktalent/PipelineHttp v0.0.0-20221004080931-279351b9fb96/go.mod h1:ob6ATP4M9FiqTRzyALSDox3kc6+xnTgzKuIT+rmKyeE=
511511
github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea h1:vuxZbB9vAwBi0Uj4F5GOfVtsi5E9MFX07EkCKypVu9M=
512512
github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea/go.mod h1:9E0C0K+/zzyJ+VqFx1llC3y7+mGgW3toLoyMQnlNXhw=
513+
github.com/hktalent/go-utils v0.0.0-20221004095234-2e23f13b429d h1:z1IUP4hqn0LGgs78bU2gSlna92/p+RlB0MSZ+RxSmCo=
514+
github.com/hktalent/go-utils v0.0.0-20221004095234-2e23f13b429d/go.mod h1:Du0lF0ZtTONXpWydjmnsL71He+zlimYLmTmAZta19ZA=
513515
github.com/hktalent/goSqlite_gorm v1.1.1 h1:kRqNFqAOtECWsUu5nBftCYFJ8MAhEHgjj2A2JOydXp0=
514516
github.com/hktalent/goSqlite_gorm v1.1.1/go.mod h1:KbxDn4W3dakhiX9eFzZ/sEAuJwKS7h42Xi3aaqU1IjE=
515517
github.com/hktalent/jarm-go v0.0.0-20220918133110-7801447b6267 h1:eH9QDUO5zwn34BLweSdpTdNcxHD/GXxxLDEG7gaR4OQ=

lib/util/asyncCmd.go

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type Cmd struct {
2323
func (cmd *Cmd) Command(name string, arg ...string) *Cmd {
2424
cmd.Cmd = exec.Command(name, arg...)
2525
cmd.stdin, _ = cmd.Cmd.StdinPipe()
26+
//cmd.stdin.Write([]byte("\n\n"))
2627
return cmd
2728
}
2829

@@ -82,38 +83,51 @@ func (r *Cmd) AsynCmd(fnCbk func(line string), szCmd string, args ...string) err
8283
if nil != err {
8384
return err
8485
}
85-
//stderr, err := cmd.StderrPipe()
86-
//if err != nil {
87-
// return err
88-
//}
89-
90-
scanner := bufio.NewScanner(cmdReader)
91-
done := make(chan struct{}, 1)
92-
go func() {
86+
done := make(chan struct{}, 2)
87+
var fnSc1 = func(bs *bufio.Scanner) {
9388
defer func() {
9489
done <- struct{}{}
9590
}()
96-
for scanner.Scan() {
91+
for bs.Scan() {
9792
select {
9893
case <-Ctx_global.Done():
9994
cmd.Exit()
10095
return
10196
default:
102-
fnCbk(scanner.Text())
97+
fnCbk(bs.Text())
10398
}
10499

105100
}
106-
}()
101+
}
102+
var bDoErr = false
103+
if bDoErr {
104+
stderr, err := cmd.StderrPipe()
105+
if err != nil {
106+
return err
107+
}
108+
scanner1 := bufio.NewScanner(stderr)
109+
go fnSc1(scanner1)
110+
}
111+
//cmd.stdin.Close()
112+
//go io.Copy(io.Discard, stderr)
113+
scanner := bufio.NewScanner(cmdReader)
114+
go fnSc1(scanner)
107115
err = cmd.Start()
108116
if err != nil {
109117
return err
110118
}
111-
<-done
112119
err = cmd.Wait()
120+
<-done
121+
if bDoErr {
122+
<-done
123+
}
113124
return err
114125
}
115126

116127
// 异步执行命令
117128
func AsynCmd(fnCbk func(line string), szCmd string, args ...string) error {
118-
return new(Cmd).AsynCmd(fnCbk, szCmd, args...)
129+
c1 := new(Cmd)
130+
err := c1.AsynCmd(fnCbk, szCmd, args...)
131+
c1.Exit()
132+
return err
119133
}

lib/util/asyncCmd_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package util
2+
3+
import (
4+
"log"
5+
"testing"
6+
)
7+
8+
func TestAsynCmd(t *testing.T) {
9+
t.Run("async cmd", func(t *testing.T) {
10+
if err := AsynCmd(func(line string) {
11+
log.Println(line)
12+
// }, "/usr/local/bin/masscan", "--max-rate", "5000", "--rate", "5000", "-p", "0-65535", "-oX", "-", "127.0.0.1"); err != nil {
13+
// }, "/bin/bash", "-i", "/Users/51pwn/MyWork/scan4all/doNmapScan.sh", "127.0.0.1"); err != nil {
14+
}, "/bin/bash", "-i", "/Users/51pwn/MyWork/scan4all/doNmapScan.sh", "127.0.0.1"); err != nil {
15+
log.Println("err ", err)
16+
}
17+
})
18+
}

lib/util/log.go

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,30 @@ func SendLog(szUrl, szVulType, Msg, Payload string) {
3838

3939
// 专门发送改造后的引擎函数执行结果
4040
func SendEngineLog(evt *models.EventData, nCurType int, data ...interface{}) {
41-
v := &SimpleVulResult{
42-
Url: evt.Task.ScanWeb,
43-
VulKind: string(Scan4all),
44-
ScanType: nCurType,
45-
ScanData: data,
41+
if nil != data && 0 < len(data) {
42+
v := &SimpleVulResult{
43+
Url: evt.Task.ScanWeb,
44+
VulKind: string(Scan4all),
45+
ScanType: nCurType,
46+
ScanData: data,
47+
}
48+
SendAnyData(v, Scan4all)
49+
writeoutput(v)
50+
}
51+
}
52+
53+
// 专门发送改造后的引擎函数执行结果
54+
func SendEngineLog4Url(Url string, nCurType int, data ...interface{}) {
55+
if nil != data && 0 < len(data) {
56+
v := &SimpleVulResult{
57+
Url: Url,
58+
VulKind: string(Scan4all),
59+
ScanType: nCurType,
60+
ScanData: data,
61+
}
62+
SendAnyData(v, Scan4all)
63+
writeoutput(v)
4664
}
47-
SendAnyData(v, Scan4all)
48-
writeoutput(v)
4965
}
5066

5167
func writeoutput(v interface{}) {

lib/util/util.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package util
22

33
import (
4+
"bufio"
45
"encoding/base64"
56
"errors"
67
"fmt"
@@ -16,6 +17,7 @@ import (
1617
"net/http"
1718
"net/url"
1819
"os"
20+
"path"
1921
"reflect"
2022
"runtime"
2123
"sort"
@@ -344,3 +346,39 @@ func CloseAll() {
344346
os.RemoveAll(GetVal(CacheName))
345347
}
346348
}
349+
350+
func RetrieveCallInfo() *map[string]interface{} {
351+
pc, file, line, _ := runtime.Caller(2)
352+
_, fileName := path.Split(file)
353+
parts := strings.Split(runtime.FuncForPC(pc).Name(), ".")
354+
pl := len(parts)
355+
packageName := ""
356+
funcName := parts[pl-1]
357+
358+
if parts[pl-2][0] == '(' {
359+
funcName = parts[pl-2] + "." + funcName
360+
packageName = strings.Join(parts[0:pl-2], ".")
361+
} else {
362+
packageName = strings.Join(parts[0:pl-1], ".")
363+
}
364+
365+
return &map[string]interface{}{
366+
"packageName": packageName,
367+
"fileName": fileName,
368+
"funcName": funcName,
369+
"line": line,
370+
}
371+
}
372+
373+
// convert bufio.Scanner to io.Reader
374+
func ScannerToReader(scanner *bufio.Scanner) io.Reader {
375+
reader, writer := io.Pipe()
376+
go func() {
377+
defer writer.Close()
378+
for scanner.Scan() {
379+
writer.Write(scanner.Bytes())
380+
}
381+
}()
382+
383+
return reader
384+
}

main.go

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,6 @@ package main
33
import (
44
"embed"
55
"fmt"
6-
"github.com/hktalent/goSqlite_gorm/lib/scan/Const"
7-
"github.com/hktalent/goSqlite_gorm/pkg/models"
8-
"github.com/hktalent/scan4all/engine"
96
"github.com/hktalent/scan4all/lib/api"
107
"github.com/hktalent/scan4all/lib/util"
118
"log"
@@ -34,7 +31,7 @@ func main() {
3431
} else {
3532
Version = util.Version
3633
}
37-
engine.Dispather(&models.Target4Chan{ScanWeb: "127.0.0.1", ScanType: Const.ScanType_Masscan})
34+
3835
szTip := ""
3936
if util.GetValAsBool("enablDevDebug") {
4037
// debug 优化时启用///////////////////////

0 commit comments

Comments
 (0)