Skip to content

Commit b6a5a8c

Browse files
x51pwnx51pwn
committed
fix httpx，vscan结果未推送到ES的bug;2、更新workflow.jpg
1 parent a187058 commit b6a5a8c

File tree

13 files changed

+60
-37
lines changed

13 files changed

+60
-37
lines changed

‎README.md

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,7 @@ scan4all -h
127127
mkdir -p logs data
128128
docker run --restart=always --ulimit nofile=65536:65536 -p 9200:9200 -p 9300:9300 -d --name es -v $PWD/logs:/usr/share/elasticsearch/logs -v $PWD/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v $PWD/config/jvm.options:/usr/share/elasticsearch/config/jvm.options -v $PWD/data:/usr/share/elasticsearch/data hktalent/elasticsearch:7.16.2
129129
# 初始化es 索引,每种工具的结果结构不一样，分开存储
130-
config/CreateEs.sh nmap
131-
config/CreateEs.sh naabu
132-
config/CreateEs.sh httpx
133-
config/CreateEs.sh nuclei
134-
config/CreateEs.sh vscan
130+
./config/initEs.sh
135131

136132
# 搜索语法，更多的查询方法，自己学 Elasticsearch
137133
http://127.0.0.1:9200/nmap_index/_doc/_search?q=_id:192.168.0.111

‎config/CreateEs.sh

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
#!/bin/bash
2+
#DELETE /${1}_index* HTTP/1.1
3+
#host:127.0.0.1:9200
4+
#User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15
5+
#Connection: keep-alive
6+
#Content-Type: application/json;charset=UTF-8
7+
#Content-Length: 0
8+
#
29
sed $'s/$/\r/' <<EOF | nc 127.0.0.1 9200
3-
DELETE /${1}_index* HTTP/1.1
4-
host:127.0.0.1:9200
5-
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15
6-
Connection: keep-alive
7-
Content-Type: application/json;charset=UTF-8
8-
Content-Length: 0
9-
1010
PUT /${1}_index HTTP/1.1
1111
host:127.0.0.1:9200
1212
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15

‎config/autoyaml.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
cd $HOME/MyWork/nuclei-templates
2+
git fetch origin master
3+
git merge origin/master

‎config/config.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,5 +60,5 @@
6060
"httpx": {},
6161
"enableEsSv": false,
6262
"esthread": 8,
63-
"esUrl": "http://127.0.0.1:9200/"
63+
"esUrl": "http://127.0.0.1:9200/%s_index/_doc/%s"
6464
}

‎config/initEs.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
config/CreateEs.sh nmap
2+
config/CreateEs.sh naabu
3+
config/CreateEs.sh httpx
4+
config/CreateEs.sh nuclei
5+
config/CreateEs.sh vscan
6+
config/CreateEs.sh hydra

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/projectdiscovery/networkpolicy v0.0.1
2121
github.com/remeh/sizedwaitgroup v1.0.0
2222
go.uber.org/ratelimit v0.2.0
23-
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
23+
golang.org/x/net v0.0.0-20220630215102-69896b714898
2424
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
2525
)
2626

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -986,6 +986,8 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su
986986
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
987987
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
988988
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
989+
golang.org/x/net v0.0.0-20220630215102-69896b714898 h1:K7wO6V1IrczY9QOQ2WkVpw4JQSwCd52UsxVEirZUfiw=
990+
golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
989991
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
990992
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
991993
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

‎pkg/httpx/runner/runner.go

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"fmt"
1010
"github.com/ammario/ipisp/v2"
1111
"github.com/hktalent/scan4all/brute"
12+
"github.com/hktalent/scan4all/pkg"
1213
"github.com/hktalent/scan4all/pkg/fingerprint"
1314
"github.com/hktalent/scan4all/pocs_go"
1415
"github.com/hktalent/scan4all/pocs_yml"
@@ -692,6 +693,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
692693
go func(target, method, protocol string) {
693694
defer wg.Done()
694695
result := r.analyze(hp, protocol, target, method, t, scanopts)
696+
go pkg.SendAnyData(result, "httpx")
695697
output <- result
696698
if scanopts.TLSProbe && result.TLSData != nil {
697699
scanopts.TLSProbe = false
@@ -746,6 +748,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
746748
defer wg.Done()
747749
h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
748750
result := r.analyze(hp, protocol, h, method, t, scanopts)
751+
go pkg.SendAnyData(result, "httpx")
749752
output <- result
750753
if scanopts.TLSProbe && result.TLSData != nil {
751754
scanopts.TLSProbe = false
@@ -1256,10 +1259,12 @@ retry:
12561259
}
12571260
return slice
12581261
}
1262+
// 登陆页面检测
12591263
if brute.CheckLoginPage(finalURL) {
12601264
technologies = append(technologies, "登录页面")
12611265
}
1262-
technologies = SliceRemoveDuplicates(technologies) // 指纹去重
1266+
// 指纹去重
1267+
technologies = SliceRemoveDuplicates(technologies)
12631268
if !scanopts.NoPOC {
12641269
intersect := func(slice1, slice2 []string) []string {
12651270
m := make(map[string]int)
@@ -1293,24 +1298,29 @@ retry:
12931298
}
12941299
return nn
12951300
}
1296-
poctechnologies1 = pocs_go.POCcheck(technologies, ul, finalURL, false) // //通过wFingerprint获取到的指纹进行检测gopoc check
1301+
//通过wFingerprint获取到的指纹进行检测gopoc check
1302+
poctechnologies1 = pocs_go.POCcheck(technologies, ul, finalURL, false)
12971303
Vullist = append(Vullist, poctechnologies1...)
12981304
for _, technology := range technologies {
12991305
pocYmlList1 := pocs_yml.Check(ul, scanopts.CeyeApi, scanopts.CeyeDomain, r.options.HTTPProxy, strings.ToLower(technology)) // 通过wFingerprint获取到的指纹进行ymlpoc check
13001306
Vullist = append(Vullist, pocYmlList1...)
13011307
}
1302-
filePaths, filefuzzTechnologies = brute.FileFuzz(ul, resp.StatusCode, resp.ContentLength, resp.Raw) // 敏感文件扫描
1308+
// 敏感文件扫描
1309+
filePaths, filefuzzTechnologies = brute.FileFuzz(ul, resp.StatusCode, resp.ContentLength, resp.Raw)
13031310
filefuzzTechnologies = SliceRemoveDuplicates(filefuzzTechnologies)
1304-
filefuzzTechnologies = difference(filefuzzTechnologies, technologies) // 取差集合
1311+
// 取差集合
1312+
filefuzzTechnologies = difference(filefuzzTechnologies, technologies)
13051313

13061314
poctechnologies2 = pocs_go.POCcheck(filefuzzTechnologies, ul, finalURL, true) //通过敏感文件扫描获取到的指纹进行检测gopoc check
13071315
Vullist = append(Vullist, poctechnologies2...)
13081316
for _, technology := range filefuzzTechnologies {
13091317
pocYmlList2 := pocs_yml.Check(ul, scanopts.CeyeApi, scanopts.CeyeDomain, r.options.HTTPProxy, strings.ToLower(technology)) //通过敏感文件扫描获取到的指纹进行检测ymlpoc check
13101318
Vullist = append(Vullist, pocYmlList2...)
13111319
}
1312-
technologies = append(technologies, filefuzzTechnologies...) // 输出加入敏感文件扫描 获取到的指纹
1313-
technologies = SliceRemoveDuplicates(technologies) // 指纹去重
1320+
// 输出加入敏感文件扫描 获取到的指纹
1321+
technologies = append(technologies, filefuzzTechnologies...)
1322+
// 指纹去重
1323+
technologies = SliceRemoveDuplicates(technologies)
13141324
}
13151325
if len(technologies) > 0 {
13161326
sort.Strings(technologies)

‎pkg/hydra/runner.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ func Start(IPAddr string, Port int, Protocol string) {
3333
var out AuthInfo
3434
for info := range crack.Out {
3535
out = info
36+
pkg.SendAData[AuthInfo](fmt.Sprintf("%s:%d", out.IPAddr, out.Port), []AuthInfo{out}, "hydra")
3637
}
3738
log.Println(out)
3839
//crack.Pool.Wait()

‎pkg/log.go

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package pkg
22

33
import (
44
"fmt"
5-
"github.com/davecgh/go-spew/spew"
65
"github.com/logrusorgru/aurora"
76
"github.com/projectdiscovery/gologger"
87
"os"
@@ -23,11 +22,6 @@ func GetPluginName(defaultVal string) string {
2322
return defaultVal
2423
}
2524

26-
// log message，Easy to send to ES result server
27-
func LogJson(logMsg interface{}) {
28-
spew.Printf("%v", logMsg)
29-
}
30-
3125
func GoPocLog(log string) {
3226
builder := &strings.Builder{}
3327
builder.WriteString("[")
@@ -39,9 +33,7 @@ func GoPocLog(log string) {
3933
builder.WriteString("] ")
4034
builder.WriteString(log)
4135
fmt.Print(builder.String())
42-
if Output != "" {
43-
writeoutput(builder.String())
44-
}
36+
writeoutput(builder.String())
4537
}
4638

4739
func YmlPocLog(log string) {
@@ -55,9 +47,7 @@ func YmlPocLog(log string) {
5547
builder.WriteString("] ")
5648
builder.WriteString(log)
5749
fmt.Print(builder.String())
58-
if Output != "" {
59-
writeoutput(builder.String())
60-
}
50+
writeoutput(builder.String())
6151
}
6252

6353
func BurteLog(log string) {
@@ -71,12 +61,14 @@ func BurteLog(log string) {
7161
builder.WriteString("] ")
7262
builder.WriteString(log)
7363
fmt.Print(builder.String())
74-
if Output != "" {
75-
writeoutput(builder.String())
76-
}
64+
writeoutput(builder.String())
7765
}
7866

7967
func writeoutput(log string) {
68+
SendAnyData(log, "vscan")
69+
if "" == Output {
70+
return
71+
}
8072
f, err := os.OpenFile(Output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
8173
if err != nil {
8274
gologger.Fatal().Msgf("Could not create output fiale '%s': %s\n", Output, err)

‎pkg/sv2es.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package pkg
22

33
import (
44
"bytes"
5+
"crypto/sha1"
6+
"encoding/hex"
57
"encoding/json"
68
"fmt"
79
"io/ioutil"
@@ -32,6 +34,17 @@ func Log(v ...any) {
3234
log.Println(v...)
3335
}
3436

37+
func SendAnyData(data interface{}, szType string) {
38+
data1, _ := json.Marshal(data)
39+
if 0 < len(data1) && "true" == GetVal("enableEsSv") {
40+
hasher := sha1.New()
41+
hasher.Write(data1)
42+
k := hex.EncodeToString(hasher.Sum(nil))
43+
go SendReq(data, k, szType)
44+
}
45+
}
46+
47+
// k is id
3548
func SendAData[T any](k string, data []T, szType string) {
3649
if 0 < len(data) && "true" == GetVal("enableEsSv") {
3750
m2 := make(map[string]interface{})
@@ -50,7 +63,7 @@ func SendReq(data1 interface{}, id, szType string) {
5063
defer func() {
5164
<-nThreads
5265
}()
53-
url := esUrl + szType + "_index/_doc/" + url.QueryEscape(id)
66+
url := fmt.Sprintf(esUrl, szType, url.QueryEscape(id))
5467
req, err := http.NewRequest("POST", url, bytes.NewReader(data))
5568
if err != nil {
5669
Log(fmt.Sprintf("%s error %v", id, err))

‎pocs_yml/check/check.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ package check
22

33
import (
44
"fmt"
5+
"github.com/hktalent/scan4all/pkg"
56
"github.com/pkg/errors"
67
"github.com/projectdiscovery/gologger"
7-
"github.com/hktalent/scan4all/pkg"
88
"io"
99
"net"
1010
"net/http"

‎static/workflow.jpg

137 KB
Loading

0 commit comments

Comments
 (0)