Skip to content

Commit b8fe42d

Browse files
hktalenthktalent
committed
1、重构了webscan
2、优化filefuzz，重用http1.1 keep-alive 3、优化，优先使用http2.0，提高效率 4、增加veo师傅的poc 2022-10-03
1 parent ea6c8bc commit b8fe42d

File tree

44 files changed

+655
-520
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+655
-520
lines changed

‎brute/dicts/filedic.txt

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5553,7 +5553,6 @@ admin%20
55535553
admin%20/
55545554
admin%20/page
55555555
admin%24
5556-
admin%ext%
55575556
admin-ANTIGO
55585557
admin-admin
55595558
admin-ajax
@@ -27949,8 +27948,8 @@ url.jsp
2794927948
url.php
2795027949
url_1.sql
2795127950
url_2.sql
27952-
urllist.
27953-
urllist.txt.gz
27951+
urlList.
27952+
urlList.txt.gz
2795427953
urlmem-app/.env
2795527954
urlogy
2795627955
urls.py

‎brute/filefuzz.go

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ func init() {
140140
var RandStr4Cookie = util.RandStringRunes(10)
141141

142142
// 重写了fuzz：优化流程、优化算法、修复线程安全bug、增加智能功能
143+
// 两次 ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
143144
func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
144145
u01, err := url.Parse(strings.TrimSpace(u))
145146
if nil == err {
@@ -187,22 +188,25 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
187188
var async_technologies = make(chan []string, util.Fuzzthreads*2)
188189
// 字典长度的 70% 的错误
189190
var MaxErrorTimes int32 = int32(float32(len(filedic)) * 0.7)
190-
defer func() {
191-
close(ch)
192-
close(async_data)
193-
close(async_technologies)
194-
}()
191+
//defer func() {
192+
// close(ch)
193+
// close(async_data)
194+
// close(async_technologies)
195+
//}()
195196
//log.Printf("start fuzz: %s for", u)
196197
nStop := 400
197198
go func() {
198199
for {
199200
select {
201+
case <-ctx2.Done():
202+
return
200203
case x1, ok := <-async_data:
201204
if ok {
202205
path = append(path, x1...)
203206
if len(path) > nStop {
204207
stop() //发停止指令
205208
atomic.AddInt32(&errorTimes, MaxErrorTimes)
209+
return
206210
}
207211
} else {
208212
return
@@ -213,13 +217,12 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
213217
} else {
214218
return
215219
}
216-
case <-ctx2.Done():
217-
return
218220
default:
219221
// <-time.After(time.Duration(100) * time.Millisecond)
220222
}
221223
}
222224
}()
225+
log.Printf("wait for file fuzz(dicts:%d) %s \r", len(filedic), u)
223226
for _, payload := range filedic {
224227
// 接收到停止信号
225228
if atomic.LoadInt32(&errorTimes) >= MaxErrorTimes {
@@ -235,14 +238,8 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
235238
wg.Done() // 控制所有线程结束
236239
<-ch // 并发控制
237240
}()
238-
//log.Printf("start file fuzz %s%s \r", u, payload)
239241
for {
240242
select {
241-
case _, ok := <-ch:
242-
if !ok {
243-
stop()
244-
return
245-
}
246243
case <-ctx.Done(): // 00-捕获所有线程关闭信号，并退出，close for all
247244
atomic.AddInt32(&errorTimes, MaxErrorTimes)
248245
return
@@ -262,9 +259,14 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
262259
szUrl = u + payload[1:]
263260
}
264261
//log.Printf("start fuzz: [%s]", szUrl)
262+
client := util.GetClient(szUrl)
263+
if nil != client {
264+
client.ErrCount = 0
265+
client.ErrLimit = 999999
266+
}
265267
if fuzzPage, req, err := reqPage(szUrl); err == nil && nil != req && 0 < len(req.Body) {
266268
//if 200 == req.StatusCode {
267-
// log.Printf("%d : %s \n", req.StatusCode, szUrl)
269+
//log.Printf("%d : %s \n", req.StatusCode, szUrl)
268270
//}
269271
go util.CheckHeader(req.Header, u)
270272
// 02-状态码和req1相同，且与req1相似度>9.5，关闭所有fuzz
@@ -324,7 +326,10 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
324326
}
325327
} else { // 这里应该元子操作
326328
if nil != err {
327-
log.Printf("%s is err %v\n", szUrl, err)
329+
//if nil != client && strings.Contains(err.Error(), " connect: connection reset by peer") {
330+
// client.Client = client.GetClient(nil)
331+
//}
332+
//log.Printf("file fuzz %s is err %v\n", szUrl, err)
328333
}
329334
atomic.AddInt32(&errorTimes, 1)
330335
}

‎config/config.json

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"OnClient": true,
23
"ScanPoolSize":5000,
34
"JndiAddress": "https://rcejndi.51pwn.com",
45
"CeyeDomain": "scan4all.51pwn.com",
@@ -63,16 +64,16 @@
6364
"HydraUser": "",
6465
"HydraPass": "",
6566
"UrlPrecise": true,
66-
"ParseSSl": false,
67-
"EnableSubfinder": false,
67+
"ParseSSl": true,
68+
"EnableSubfinder": true,
6869
"EnableHoneyportDetection": true,
6970
"EnableKsubdomain": true,
7071
"KsubdomainRegxp": "([0-9a-zA-Z\\-]+\\.[0-9a-zA-Z\\-]+)$",
7172
"naabu_dns": {},
7273
"naabu": {"TopPorts": "1000","ScanAllIPS": true,"Threads": 50,"EnableProgressBar": false},
73-
"priorityNmap": false,
74+
"priorityNmap": true,
7475
"noScan": false,
75-
"enableMultNuclei": false,
76+
"enableMultNuclei": true,
7677
"enableNuclei": true,
7778
"nuclei": {
7879
"Severities": [5,4,3],
@@ -86,7 +87,7 @@
8687
"HeadlessTemplateThreads": 10,
8788
"ReportingConfig": "config/nuclei_esConfig.yaml"
8889
},
89-
"enablDevDebug": false,
90+
"enablDevDebug": true,
9091
"enablEmbedYaml": true,
9192
"enablFileFuzz": true,
9293
"httpx": {
@@ -100,7 +101,7 @@
100101
"MaxRedirects": 3
101102
},
102103
"enableEsSv": false,
103-
"CheckWeakPassword": true,
104+
"CheckWeakPassword": false,
104105
"esthread": 8,
105106
"hydrathread": 64,
106107
"Fuzzthreads": 16,

‎config/databases/db_drupal

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5656,7 +5656,7 @@
56565656
"515658","urlfilter"
56575657
"515659","urlicon"
56585658
"515660","urlify"
5659-
"515661","urllist"
5659+
"515661","urlList"
56605660
"515662","urlproxy"
56615661
"515663","usability_suite"
56625662
"515664","uscongress"

‎engine/dispather.go

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,71 @@ package engine
33
import (
44
"github.com/hktalent/goSqlite_gorm/lib"
55
"github.com/hktalent/goSqlite_gorm/lib/scan/Const"
6+
"github.com/hktalent/goSqlite_gorm/pkg/models"
7+
"github.com/hktalent/scan4all/pkg/portScan"
68
)
79

8-
func Dispather(task *lib.Target4Chan) {
10+
var (
11+
CaseScanFunc = map[int]interface{}{
12+
Const.ScanType_SSLInfo: nil, // 01- SSL信息分析，并对域名信息进行收集、进入下一步流程
13+
Const.ScanType_SubDomain: nil, // 02- 子域名爆破，新域名回归 到: 1 <-- -> 2，做去重处理
14+
Const.ScanType_MergeIps: nil, // 03- 默认自动合并ip，记录ip与域名的关联关系，再发送payload时考虑：相同ip不同域名，相同payload分别发送 合并相同目标 若干域名的ip，避免扫描时重复
15+
Const.ScanType_Pswd4hydra: nil, // 04- 密码破解，隐含包含了: 端口扫描(05-masscan + 06-nmap)
16+
Const.ScanType_Masscan: portScan.MassScanTarget, // 05- 合并后的ip 进行快速端口扫描
17+
Const.ScanType_Nmap: nil, // 06、精准 端口指纹，排除masscan已经识别的几种指纹
18+
Const.ScanType_IpInfo: nil, // 07- 获取ip info
19+
Const.ScanType_GoPoc: nil, // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
20+
Const.ScanType_PortsWeb: nil, // 09- web端口识别，Naabu,识别 https，识别存活的web端口，再进入下一流程
21+
Const.ScanType_WebFingerprints: nil, // 10- web指纹，识别蜜罐，并标识
22+
Const.ScanType_WebDetectWaf: nil, // 11- detect WAF
23+
Const.ScanType_WebScrapy: nil, // 12- 爬虫分析，form表单识别，字段名识别，form action提取；
24+
Const.ScanType_WebInfo: nil, // 13- server、x-powerby、x***，url、ip、其他敏感信息（姓名、电话、地址、身份证）
25+
Const.ScanType_WebVulsScan: nil, // 14-nuclei
26+
Const.ScanType_WebDirScan: nil, // 14-dir爆破,Gobuster
27+
}
28+
)
929

10-
switch Const.d {
30+
// 扫描任务分发
31+
// 为不同类型扫描构造参数，进行事件分发
32+
func Dispather(task *models.Target4Chan) {
33+
for k, _ := range CaseScanFunc {
34+
if lib.HasScanType(task.ScanType, k) {
35+
x1 := &models.EventData{EventType: k}
36+
switch k {
37+
case Const.ScanType_SSLInfo: // 01- SSL信息分析，并对域名信息进行收集、进入下一步流程
38+
G_Engine.EventData <- x1
39+
case Const.ScanType_SubDomain: // 02- 子域名爆破，新域名回归 到: 1 <-- -> 2，做去重处理
40+
G_Engine.EventData <- x1
41+
case Const.ScanType_MergeIps: // 03- 默认自动合并ip，记录ip与域名的关联关系，再发送payload时考虑：相同ip不同域名，相同payload分别发送 合并相同目标 若干域名的ip，避免扫描时重复
42+
G_Engine.EventData <- x1
43+
case Const.ScanType_Pswd4hydra: // 04- 密码破解，隐含包含了: 端口扫描(05-masscan + 06-nmap)
44+
x1.EventData = []interface{}{x1, []interface{}{portScan.TargetStr(task.ScanWeb)}}
45+
G_Engine.EventData <- x1
46+
case Const.ScanType_Masscan: // 05- 合并后的ip 进行快速端口扫描
47+
G_Engine.EventData <- x1
48+
case Const.ScanType_Nmap: // 06、精准 端口指纹，排除masscan已经识别的几种指纹
49+
G_Engine.EventData <- x1
50+
case Const.ScanType_IpInfo: // 07- 获取ip info
51+
G_Engine.EventData <- x1
52+
case Const.ScanType_GoPoc: // 08- go-poc 检测, 隐含包含了: 端口扫描(05-masscan + 06-nmap)
53+
G_Engine.EventData <- x1
54+
case Const.ScanType_PortsWeb: // 09- web端口识别，Naabu,识别 https，识别存活的web端口，再进入下一流程
55+
G_Engine.EventData <- x1
56+
case Const.ScanType_WebFingerprints: // 10- web指纹，识别蜜罐，并标识
57+
G_Engine.EventData <- x1
58+
case Const.ScanType_WebDetectWaf: // 11- detect WAF
59+
G_Engine.EventData <- x1
60+
case Const.ScanType_WebScrapy: // 12- 爬虫分析，form表单识别，字段名识别，form action提取；
61+
G_Engine.EventData <- x1
62+
case Const.ScanType_WebInfo: // 13- server、x-powerby、x***，url、ip、其他敏感信息（姓名、电话、地址、身份证）
63+
G_Engine.EventData <- x1
64+
case Const.ScanType_WebVulsScan: // 14-nuclei
65+
G_Engine.EventData <- x1
66+
case Const.ScanType_WebDirScan: // 14-dir爆破,Gobuster
67+
G_Engine.EventData <- x1
68+
default:
1169

70+
}
71+
}
1272
}
1373
}

‎engine/engine.go

Lines changed: 25 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,21 @@ package engine
33
import (
44
"context"
55
"github.com/codegangsta/inject"
6+
"github.com/hktalent/goSqlite_gorm/pkg/models"
67
"github.com/hktalent/scan4all/lib/util"
7-
"github.com/hktalent/scan4all/pkg/portScan"
88
"github.com/hktalent/scan4all/pocs_go"
99
"github.com/panjf2000/ants/v2"
1010
"log"
1111
"sync"
1212
)
1313

14-
// 事件数据
15-
type EventData struct {
16-
EventType string // 类型：masscan、nmap、
17-
EventData []interface{} // func，parms
18-
}
19-
2014
// 引擎对象，全局单实例
2115
type Engine struct {
22-
Context *context.Context // 上下文
23-
Wg *sync.WaitGroup // Wg
24-
Pool int // 线程池
25-
PoolFunc *ants.PoolWithFunc // 线程调用
26-
EventData chan *EventData // 数据队列
16+
Context *context.Context // 上下文
17+
Wg *sync.WaitGroup // Wg
18+
Pool int // 线程池
19+
PoolFunc *ants.PoolWithFunc // 线程调用
20+
EventData chan *models.EventData // 数据队列
2721
}
2822

2923
// 全局引擎
@@ -37,11 +31,11 @@ func NewEngine(c *context.Context, pool int) *Engine {
3731
if nil != G_Engine {
3832
return G_Engine
3933
}
40-
G_Engine = &Engine{Context: c, Wg: util.Wg, Pool: pool, EventData: make(chan *EventData, pool)}
34+
G_Engine = &Engine{Context: c, Wg: util.Wg, Pool: pool, EventData: make(chan *models.EventData, pool)}
4135

4236
p, err := ants.NewPoolWithFunc(pool, func(i interface{}) {
4337
defer G_Engine.Wg.Done()
44-
G_Engine.DoEvent(i.(*EventData))
38+
G_Engine.DoEvent(i.(*models.EventData))
4539
})
4640
if nil != err {
4741
log.Println("ants.NewPoolWithFunc is error: ", err)
@@ -59,16 +53,9 @@ func (e *Engine) Close() {
5953
}
6054

6155
// case 扫描使用的函数
62-
func (e *Engine) DoCase(ed *EventData) interface{} {
63-
if nil != ed {
64-
switch ed.EventType {
65-
case "masscan":
66-
return portScan.ScanTarget
67-
case "nmap":
68-
return portScan.ScanTarget
69-
default:
70-
return nil
71-
}
56+
func (e *Engine) DoCase(ed *models.EventData) interface{} {
57+
if i, ok := CaseScanFunc[ed.EventType]; ok {
58+
return i
7259
}
7360
return nil
7461
}
@@ -77,7 +64,10 @@ func (e *Engine) DoCase(ed *EventData) interface{} {
7764
// 每个事件自己做防重处理
7865
// 每个事件异步执行
7966
// 每种事件类型可以独立控制并发数
80-
func (e *Engine) DoEvent(ed *EventData) {
67+
func (e *Engine) DoEvent(ed *models.EventData) {
68+
var x01 = &models.EventData{}
69+
if nil != x01 {
70+
}
8171
if nil != ed {
8272
fnCall := e.DoCase(ed)
8373
if nil != fnCall {
@@ -87,7 +77,7 @@ func (e *Engine) DoEvent(ed *EventData) {
8777
}
8878
v, err := in.Invoke(fnCall)
8979
if nil != err {
90-
log.Printf("DoEvent %s is error: %v %+v \n", ed.EventType, err, ed.EventData)
80+
log.Printf("DoEvent %d is error: %v %+v \n", ed.EventType, err, ed.EventData)
9181
} else if nil != v {
9282
log.Printf("DoEvent result %s %v\n", ed.EventType, v)
9383
}
@@ -102,19 +92,23 @@ func init() {
10292
util.RegInitFunc(func() {
10393
x1 := NewEngine(&util.Ctx_global, util.GetValAsInt("ScanPoolSize", 5000))
10494
// 异步启动一个线程处理检测，避免
95+
util.Wg.Add(1)
10596
go func() {
106-
defer x1.Close()
97+
defer func() {
98+
x1.Close()
99+
util.Wg.Done()
100+
}()
107101
//nMax := 120 // 等xxx秒都没有消息进入就退出
108102
//nCnt := 0
109103
for {
110104
select {
111105
case <-util.Ctx_global.Done():
112106
close(util.PocCheck_pipe)
113107
return
114-
case x1, ok := <-G_Engine.EventData: // 各种扫描的控制
115-
if ok {
108+
case x2 := <-G_Engine.EventData: // 各种扫描的控制
109+
if nil != x2 {
116110
G_Engine.Wg.Add(1)
117-
G_Engine.PoolFunc.Invoke(x1)
111+
G_Engine.PoolFunc.Invoke(x2)
118112
}
119113
case x1, ok := <-util.PocCheck_pipe:
120114
if util.GetValAsBool("NoPOC") || nil == x1 || !ok {
@@ -131,6 +125,7 @@ func init() {
131125
})
132126
default:
133127
util.DoDelayClear()
128+
util.DoSleep()
134129
//var f01 float32 = float32(nCnt) / float32(nMax) * float32(100)
135130
//fmt.Printf(" Asynchronous go PoCs detection task %%%0.2f ....\r", f01)
136131
//<-time.After(time.Duration(1) * time.Second)

‎engine/tools.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package engine

‎fixMod.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,7 @@ git checkout vendor/github.com/projectdiscovery/nuclei/v2
1313
go build
1414

1515
git add vendor
16+
rm -rf vendor/github.com/hktalent/goSqlite_gorm
17+
ln -s $HOME/MyWork/goSqlite_gorm $PWD/vendor/github.com/hktalent/goSqlite_gorm
18+
19+

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ require (
100100
github.com/google/go-github v17.0.0+incompatible
101101
github.com/gorilla/websocket v1.5.0
102102
github.com/gosnmp/gosnmp v1.35.0
103+
github.com/hktalent/PipelineHttp v0.0.0-20221003131356-bf29cf889d54
103104
github.com/hktalent/goSqlite_gorm v1.1.0
104105
github.com/hktalent/jarm-go v0.0.0-20220918133110-7801447b6267
105106
github.com/huin/asn1ber v0.0.0-20120622192748-af09f62e6358

0 commit comments

Comments
 (0)