Skip to content

Commit 16e8b29

Browse files
hktalenthktalent
committed
- 2022-10-03 优化了fuzz，http2.0下测试18秒可以完成6万的扫描，同时合并、去除冗余的结果
- 2022-10-03 优化：所有的web扫描前，均做有效检测，避免无效扫描，提升了效率 2022-10-07
1 parent 00ada23 commit 16e8b29

File tree

22 files changed

+111
-64
lines changed

22 files changed

+111
-64
lines changed

‎README_CN.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,8 @@ priorityNmap=false ./scan4all -tp http -list allOut.txt -v
207207
more see: <a href=https://github.com/hktalent/scan4all/discussions>discussions</a>
208208

209209
# 变更日志
210+
- 2022-10-03 优化了fuzz，http2.0下测试18秒可以完成6万的扫描，同时合并、去除冗余的结果
211+
- 2022-10-03 优化：所有的web扫描前，均做有效检测，避免无效扫描，提升了效率
210212
- 2022-07-28 为 nuclei 添加 substr、 aes_cbc DSL 函数<a href="https://github.com/projectdiscovery/nuclei/releases/tag/v2.7.7">nuclei v2.7.7</a>
211213
- 2022-08-03 fixed nuclei Multiple instances cache goroutine leaks PR<a href=https://github.com/projectdiscovery/nuclei/issues/2386>#2386</a>
212214
- 2022-07-20 fix and PR nuclei <a href=https://github.com/projectdiscovery/nuclei/issues/2301>#2301</a> 并发多实例的bug

‎brute/dicts/fuzz404.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
403
33
<a href="/#/error/404">Found</a>
44
404
5+
404 page not found
56
404 Not Found
67
404.safedog.cn
78
Access Failed

‎brute/filefuzz.go

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,11 @@ var FileFuzz4Engin = util.EngineFuncFactory(func(evt *models.EventData, args ...
147147
util.SendEngineLog(evt, Const.ScanType_WebDirScan, filePaths, fileFuzzTechnologies)
148148
})
149149

150+
type FuzzData struct {
151+
Path *[]string
152+
Req *util.Page
153+
}
154+
150155
// 重写了fuzz：优化流程、优化算法、修复线程安全bug、增加智能功能
151156
// 两次 ioutil.ReadAll(resp.Body)，第二次就会 Read返回EOF error
152157
func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody string) ([]string, []string) {
@@ -195,7 +200,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
195200
// 控制 fuzz 线程数
196201
var ch = make(chan struct{}, util.Fuzzthreads)
197202
// 异步接收结果
198-
var async_data = make(chan []string, util.Fuzzthreads*2)
203+
var async_data = make(chan *FuzzData, util.Fuzzthreads*2)
199204
var async_technologies = make(chan []string, util.Fuzzthreads*2)
200205
// 字典长度的 70% 的错误
201206
var MaxErrorTimes int32 = int32(float32(len(filedic)) * 0.7)
@@ -209,14 +214,18 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
209214
//}()
210215
//log.Printf("start fuzz: %s for", u)
211216
nStop := 400
217+
var lst200 *util.Response
212218
go func() {
213219
for {
214220
select {
215221
case <-ctx2.Done():
216222
return
217223
case x1, ok := <-async_data:
218224
if ok {
219-
path = append(path, x1...)
225+
if lst200 == nil || x1.Req.Resqonse.Body != lst200.Body {
226+
path = append(path, (*x1.Path)...)
227+
}
228+
lst200 = x1.Req.Resqonse
220229
if len(path) > nStop {
221230
stop() //发停止指令
222231
atomic.AddInt32(&errorTimes, MaxErrorTimes)
@@ -237,7 +246,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
237246
}
238247
}()
239248
log.Printf("wait for file fuzz(dicts:%d) %s \r", len(filedic), u)
240-
var lst200 *util.Response
249+
241250
for _, payload := range filedic {
242251
// 接收到停止信号
243252
if atomic.LoadInt32(&errorTimes) >= MaxErrorTimes {
@@ -315,7 +324,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
315324
a11 := ByPass403(&u, &payload, &wg)
316325
// 表示 ByPass403 成功了, 结果、控制台输出点什么？
317326
if 0 < len(a11) {
318-
async_data <- a11
327+
async_data <- &FuzzData{Path: &a11, Req: fuzzPage}
319328
}
320329
}
321330
return
@@ -339,7 +348,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
339348
path1 = append(path1, *fuzzPage.Url)
340349
}
341350
if 0 < len(path1) {
342-
async_data <- path1
351+
async_data <- &FuzzData{Path: &path1, Req: fuzzPage}
343352
}
344353
if 0 < len(technologies1) {
345354
async_technologies <- technologies1
@@ -360,7 +369,7 @@ func FileFuzz(u string, indexStatusCode int, indexContentLength int, indexbody s
360369
}
361370
// 默认情况等待所有结束
362371
wg.Wait()
363-
log.Printf("fuzz is over: %s\n", u)
372+
log.Printf("fuzz is over: %s found:\n%s\n", u, strings.Join(path, "\n"))
364373
technologies = util.SliceRemoveDuplicates(technologies)
365374
path = util.SliceRemoveDuplicates(path)
366375
stop() //发停止指令

‎config/scan4all_db.db

224 KB
Binary file not shown.

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/remeh/sizedwaitgroup v1.0.0
2121
go.uber.org/ratelimit v0.2.0
2222
golang.org/x/net v0.0.0-20221004154528-8021a29435af
23-
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec
23+
golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875
2424
)
2525

2626
require (
@@ -101,7 +101,7 @@ require (
101101
github.com/google/go-github v17.0.0+incompatible
102102
github.com/gorilla/websocket v1.5.0
103103
github.com/gosnmp/gosnmp v1.35.0
104-
github.com/hktalent/PipelineHttp v0.0.0-20221006120713-f0d2d692285e
104+
github.com/hktalent/PipelineHttp v0.0.0-20221007020748-952ec7255bdb
105105
github.com/hktalent/goSqlite_gorm v1.1.1
106106
github.com/hktalent/jarm-go v0.0.0-20220918133110-7801447b6267
107107
github.com/huin/asn1ber v0.0.0-20120622192748-af09f62e6358
@@ -313,7 +313,7 @@ require (
313313
go.uber.org/multierr v1.8.0 // indirect
314314
go.uber.org/zap v1.23.0 // indirect
315315
goftp.io/server/v2 v2.0.0 // indirect
316-
golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 // indirect
316+
golang.org/x/exp v0.0.0-20221006183845-316c7553db56 // indirect
317317
golang.org/x/mod v0.6.0-dev.0.20221005201717-2666ed6287c1 // indirect
318318
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
319319
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect

‎go.sum

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,8 @@ github.com/hktalent/PipelineHttp v0.0.0-20221006102853-7270ca9cc3dc h1:dz5vNFzfG
542542
github.com/hktalent/PipelineHttp v0.0.0-20221006102853-7270ca9cc3dc/go.mod h1:ncw1+ugTc5GPQLUHHI7uWrgW2KWBppDBWwwjC984QJg=
543543
github.com/hktalent/PipelineHttp v0.0.0-20221006120713-f0d2d692285e h1:6Iy5XhPWznVIQEXJNFeE/RyXe3wrIfIjybW/yLAbch4=
544544
github.com/hktalent/PipelineHttp v0.0.0-20221006120713-f0d2d692285e/go.mod h1:ncw1+ugTc5GPQLUHHI7uWrgW2KWBppDBWwwjC984QJg=
545+
github.com/hktalent/PipelineHttp v0.0.0-20221007020748-952ec7255bdb h1:u639eutmwOJ1eBdyqvmCCq96v6tU1+b/PwIf1uaJvkE=
546+
github.com/hktalent/PipelineHttp v0.0.0-20221007020748-952ec7255bdb/go.mod h1:ncw1+ugTc5GPQLUHHI7uWrgW2KWBppDBWwwjC984QJg=
545547
github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea h1:vuxZbB9vAwBi0Uj4F5GOfVtsi5E9MFX07EkCKypVu9M=
546548
github.com/hktalent/go-utils v0.0.0-20221004021941-7e10e0fb13ea/go.mod h1:9E0C0K+/zzyJ+VqFx1llC3y7+mGgW3toLoyMQnlNXhw=
547549
github.com/hktalent/go-utils v0.0.0-20221004095234-2e23f13b429d h1:z1IUP4hqn0LGgs78bU2gSlna92/p+RlB0MSZ+RxSmCo=
@@ -1279,6 +1281,8 @@ golang.org/x/exp v0.0.0-20221002003631-540bb7301a08 h1:LtBIgSqNhkuC9gA3BFjGy5obH
12791281
golang.org/x/exp v0.0.0-20221002003631-540bb7301a08/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
12801282
golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 h1:fGZugkZk2UgYBxtpKmvub51Yno1LJDeEsRp2xGD+0gY=
12811283
golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
1284+
golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ=
1285+
golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
12821286
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
12831287
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
12841288
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1483,6 +1487,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
14831487
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
14841488
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
14851489
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1490+
golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 h1:AzgQNqF+FKwyQ5LbVrVqOcuuFB67N47F9+htZYH0wFM=
1491+
golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
14861492
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
14871493
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
14881494
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

‎lib/crawlergo/mychromedp.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ func (r *MyChromedp) DoUrlWithFlg(szUrl string, head *map[string]interface{}, ti
154154
//if err := ioutil.WriteFile("screenshot1.png", b1, 0o644); err != nil {
155155
// log.Fatal(err)
156156
//}
157-
log.Println(title)
157+
//log.Println(title)
158158
//c1 := chromedp.FromContext(taskCtx)
159159
return nil, cancel
160160
}

‎lib/util/asyncCmd.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package util
33
import (
44
"bufio"
55
"io"
6-
"log"
76
"os"
87
"os/exec"
98
"time"
@@ -77,7 +76,7 @@ func (r *Cmd) WriteInput(args ...string) {
7776

7877
func (r *Cmd) AsynCmd(fnCbk func(line string), szCmd string, args ...string) error {
7978
cmd := r.Command(szCmd, args...)
80-
log.Println(cmd.Args)
79+
//log.Println(cmd.Args)
8180
var err error
8281
cmdReader, err := cmd.StdoutPipe()
8382
if nil != err {

‎lib/util/config.go

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ var TmpFile = map[string][]*os.File{}
113113
func GetTempFile(t string) *os.File {
114114
tempInput, err := ioutil.TempFile("", "scan4all-out*")
115115
if err != nil {
116-
log.Println(err)
116+
//log.Println(err)
117117
return nil
118118
} else {
119119
if t1, ok := TmpFile[t]; ok {
@@ -438,29 +438,29 @@ func TestIs404(szUrl string) (r01 *Response, err error, ok bool) {
438438
}
439439
}
440440
sz404 := szUrl + Abs404
441-
client := GetClient(sz404)
442-
if nil != client {
443-
//client.Client.Timeout = 500
444-
//client.ErrCount = 0
445-
//client.ErrLimit = 9999
446-
//log.Printf("%v %s \n", client, sz404)
447-
}
441+
//client := GetClient(sz404)
442+
//if nil != client {
443+
// client.Client.Timeout = 500
444+
// client.ErrCount = 0
445+
// //client.ErrLimit = 9999
446+
// //log.Printf("%v %s \n", client, sz404)
447+
//}
448448

449449
//log.Println("start test ", sz404)
450450
var mh1 map[string]string
451-
if strings.HasPrefix(sz404, "http://") {
452-
mh1 = map[string]string{
453-
//"Connection": "close",
454-
"Content-Type": "",
455-
}
456-
}
451+
//if strings.HasPrefix(sz404, "http://") {
452+
// mh1 = map[string]string{
453+
// //"Connection": "close",
454+
// "Content-Type": "",
455+
// }
456+
//}
457457
r01, err = HttpRequset(sz404, "GET", "", false, mh1)
458458
ok = err == nil && nil != r01 && 404 == r01.StatusCode
459459
if nil != err {
460460
CloseHttpClient(sz404)
461461
//log.Println(sz404, err)
462462
} else {
463-
log.Printf("%d %s %s\n", r01.StatusCode, r01.Protocol, sz404)
463+
//log.Printf("%d %s %s\n", r01.StatusCode, r01.Protocol, sz404)
464464
}
465465
noRpt.Set(key, []interface{}{r01, err, ok}, defaultInteractionDuration)
466466
//client.Client.Timeout = 10

‎lib/util/db.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ func InitDb(dst ...interface{}) *gorm.DB {
7272
log.Println("sqlite db init Connection failed", err)
7373
}
7474
} else {
75-
log.Println(err)
75+
//log.Println(err)
7676
}
7777
return dbCC
7878
}

‎lib/util/sv2es.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func SendReq(data1 interface{}, id string, szType ESaveType) {
105105
body, err := ioutil.ReadAll(resp.Body)
106106
if nil == err && 0 < len(body) {
107107
Log("Es save result ", string(body))
108-
} else {
108+
} else if nil != err {
109109
Log(err)
110110
}
111111
}

‎main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ var Version string
2323
func main() {
2424
//os.Args = []string{"", "-host", "http://192.168.0.109", "-v"}
2525
//os.Args = []string{"", "-host", "http://127.0.0.1", "-v"}
26+
//os.Args = []string{"", "-list", "7b8fa7a85f9f6ae6f9178504d2202666fb8dc772.xml", "-v"}
27+
2628
runtime.GOMAXPROCS(runtime.NumCPU())
2729
util.DoInit(&config)
2830
// set version

‎pkg/http2/client.go

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎pkg/httpx/runner/options.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ func ParseOptions() *Options {
273273
options.OutputMatchString = ""
274274
options.OutputMatchRegex = ""
275275
options.OutputExtractRegex = ""
276-
options.OutputFilterStatusCode = ""
276+
options.OutputFilterStatusCode = "400,404,500"
277277
options.OutputFilterContentLength = ""
278278
options.OutputFilterLinesCount = ""
279279
options.OutputFilterWordsCount = ""

‎pkg/naabu/v2/pkg/runner/targets.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,12 +190,12 @@ func (r *Runner) DoTargets() (bool, error) {
190190
s009 = "/config/doNmapScanWin.bat "
191191
}
192192
x := util.SzPwd + s009 + r.targetsFile + " " + tempInput1.Name()
193-
log.Println(x)
193+
//log.Println(x)
194194
ss, err := util.DoCmd(strings.Split(x, " ")...)
195195
s0 := tempInput1.Name()
196196
if nil == err {
197197
if "" != ss {
198-
log.Println(ss, "\n")
198+
// log.Println(ss, "\n")
199199
}
200200
if util.FileExists(s0) {
201201
//data, err := tempInput1.Stat()

‎pkg/portScan/masscan.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func (m *Masscan) Run(fnCbk func(*models.Host)) error {
7979
err := util.AsynCmd(func(line string) {
8080
x1, err := m.ParseLine(line)
8181
if nil != err {
82-
log.Println(err)
82+
//log.Println(err)
8383
return
8484
}
8585
for _, i := range x1 {

‎pkg/portScan/nmapScan.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func (s *Scanner) scan(nmapScanner *nmap.Scanner, fnCbk func(*Stream)) ([]*Strea
105105
scanner1 := nmapScanner.GetStdout()
106106
for scanner1.Scan() {
107107
s091 := scanner1.Text()
108-
log.Println(s091)
108+
//log.Println(s091)
109109
if r09, err := nmap.Parse([]byte(s091)); nil == err {
110110
for _, host := range r09.Hosts {
111111
if len(host.Ports) == 0 || len(host.Addresses) == 0 {
@@ -130,7 +130,7 @@ func (s *Scanner) scan(nmapScanner *nmap.Scanner, fnCbk func(*Stream)) ([]*Strea
130130
}
131131
}
132132
} else {
133-
log.Println(err)
133+
//log.Println(err)
134134
}
135135
}
136136
log.Printf("Found %d Real Time Streaming Protocol (RTSP)\n", len(streams))

‎projectdiscovery/uncover/uncover.go

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,39 @@ package uncover
22

33
import (
44
"context"
5+
"strings"
56

67
// Attempts to increase the OS file descriptors - Fail silently
78
_ "github.com/projectdiscovery/fdmax/autofdmax"
89
"github.com/projectdiscovery/gologger"
910
"github.com/projectdiscovery/uncover/runner"
1011
)
1112

12-
func DoUncover() {
13+
// https://github.com/projectdiscovery/uncover
14+
/*
15+
Query multiple search engine at once
16+
Available Search engine support
17+
Shodan
18+
Censys
19+
FOFA
20+
Hunter
21+
Quake
22+
Zoomeye
23+
*/
24+
func DoUncover(targets []string) {
1325
// Parse the command line flags and read config files
14-
options := runner.ParseOptions()
26+
options := &runner.Options{Provider: &runner.Provider{},
27+
Query: targets,
28+
Engine: strings.Split("shodan,shodan-idb,fofa,censys", ","),
29+
Timeout: 30,
30+
Delay: 1,
31+
JSON: true,
32+
Limit: 10000,
33+
NoColor: true,
34+
Silent: true,
35+
Version: false,
36+
Verbose: false,
37+
}
1538

1639
newRunner, err := runner.NewRunner(options)
1740
if err != nil {

0 commit comments

Comments
 (0)