fix 1、集成Elasticsearch存储中间结果 2、嵌入整个config目录到程序中 2022-06-30 23:00:1656601220

Author: x51pwn

README.md

@@ -22,6 +22,7 @@
 - 轻量级、开源、跨平台使用
 - 支持多种类型的输入 - STDIN/HOST/IP/CIDR/URL/TXT
 - 支持多种输出类型 - JSON/TXT/CSV/STDOUT
+- 可配置将结果统一存储到Elasticsearch
 
 ## 由配置文件、环境变量控制的新特性
 - 带有上下文路径的url列表，启用精确扫描 UrlPrecise=true ./main -l xx.txt
@@ -93,7 +94,10 @@
   "naabu_dns": {},  // naabu工具对dns配置
   "naabu": {"TopPorts": "1000","ScanAllIPS": true}, // naabu配置
   "nuclei": {}, // nuclei配置，例如线程等
-  "httpx": {} // httpx 配置
+  "httpx": {}   // httpx 配置,
+  "enableEsSv": true,        // 开启结果send 到es
+  "esthread": 8 // 结果写入Elasticsearch的线程数,
+  "esUrl": "http://127.0.0.1:9200/" // Elasticsearch url
 }
 ```
 
@@ -118,7 +122,22 @@ go install github.com/hktalent/[email protected]
 scan4all -h
 ```
 # 如何使用
-使用前请自行安装nmap
+- 1、启动 Elasticsearch, 当然你可以使用传统方式输出、结果
+```bash
+mkdir -p logs data
+docker run --restart=always --ulimit nofile=65536:65536 -p 9200:9200 -p 9300:9300 -d --name es -v $PWD/logs:/usr/share/elasticsearch/logs -v $PWD/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v $PWD/config/jvm.options:/usr/share/elasticsearch/config/jvm.options  -v $PWD/data:/usr/share/elasticsearch/data  hktalent/elasticsearch:7.16.2
+# 初始化es 索引
+config/CreateEs.sh nmap
+config/CreateEs.sh naabu
+config/CreateEs.sh httpx
+config/CreateEs.sh nuclei
+
+# 搜索语法，更多的查询方法，自己学 Elasticsearch
+http://127.0.0.1:9200/nmap_index/_doc/_search?q=92.168.0.111
+其中92.168.0.111 是要查询的目标
+
+```
+- 使用前请自行安装nmap
 <a href=https://github.com/hktalent/scan4all/discussions>使用帮助</a>
 ```bash
 go build

config/CreateEs.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+sed $'s/$/\r/' <<EOF | nc 127.0.0.1 9200
+DELETE /${1}_index* HTTP/1.1
+host:127.0.0.1:9200
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15
+Connection: keep-alive
+Content-Type: application/json;charset=UTF-8
+Content-Length: 0
+
+PUT /${1}_index HTTP/1.1
+host:127.0.0.1:9200
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15
+Connection: keep-alive
+Content-Type: application/json;charset=UTF-8
+Content-Length: 405
+
+{
+  "settings": {
+   "analysis": {
+     "analyzer": {
+       "default": {
+         "type": "custom",
+         "tokenizer": "ik_smart",
+         "char_filter": [
+            "html_strip"
+          ]
+       },
+       "default_search": {
+         "type": "custom",
+         "tokenizer": "ik_smart",
+         "char_filter": [
+            "html_strip"
+          ]
+      }
+     }
+   }
+  }
+}
+PUT /${1}_index/_settings HTTP/1.1
+host:127.0.0.1:9200
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15
+Connection: close
+Content-Type: application/json;charset=UTF-8
+Content-Length: 171
+
+{
+    "index.translog.durability": "async",
+    "index.translog.sync_interval": "5s",
+    "index.translog.flush_threshold_size":"100m",
+   "refresh_interval": "30s"
+}
+
+EOF
+
+xxx=$(/usr/bin/curl -s -k -q http://localhost:9200/${1}_index/_settings &2>/dev/null)
+echo  $xxx|jq

config/config.json

@@ -56,5 +56,9 @@
   "naabu_dns": {},
   "naabu": {"TopPorts": "1000","ScanAllIPS": true},
   "nuclei": {},
-  "httpx": {}
+  "enablEmbedYaml": false,
+  "httpx": {},
+  "enableEsSv": true,
+  "esthread": 8,
+  "esUrl": "http://127.0.0.1:9200/"
 }

config/elasticsearch.yml

@@ -0,0 +1,111 @@
+#集群名称
+cluster.name: my-application
+#节点名称
+node.name: node-1
+#数据和日志的存储目录
+path.data: /usr/share/elasticsearch/data
+path.logs: /usr/share/elasticsearch/logs
+#设置绑定的ip，设置为0.0.0.0以后就可以让任何计算机节点访问到了
+network.host: 0.0.0.0
+transport.host: 0.0.0.0
+network.publish_host: 192.168.0.107
+#端口
+http.port: 9200
+# 设置在集群中的所有节点名称，这个节点名称就是之前所修改的，当然你也可以采用默认的，目前是单机，放入一个节点即可
+discovery.seed_hosts: [ "192.168.0.112:9300","192.168.0.107:9301","192.168.0.107:9302", "192.168.0.107:9300"]
+cluster.initial_master_nodes: [ "192.168.0.112:9300","192.168.0.107:9301","192.168.0.107:9302", "192.168.0.107:9300"]
+cluster.routing.allocation.same_shard.host: true
+discovery.zen.fd.ping_timeout: 1m
+discovery.zen.fd.ping_retries: 5
+
+# 设置这个参数来保证集群中的节点可以知道其它N个有master资格的节点。默认为1，对于大的集群来说，可以设置大一点的值（2-4）
+# discovery.zen.minimum_master_nodes: 1
+# node.master: true
+# node.ingest: true
+# node.data: true
+
+# cluster.routing.allocation.node_initial_primaries_recoveries: 8
+# cluster.routing.allocation.node_concurrent_recoveries: 2
+# discovery.zen.fd.ping_interval: 10s
+# discovery.zen.fd.ping_timeout: 120s
+# discovery.zen.fd.ping_retries: 6
+
+# thread_pool.get.size: 16
+# # must be <= 5
+# thread_pool.write.size: 4
+# thread_pool.search.size: 8
+# thread_pool.get.queue_size: 8
+# thread_pool.write.queue_size: 4
+# thread_pool.search.queue_size: 4
+# thread_pool.listener.queue_size: 8
+# thread_pool.analyze.queue_size: 8
+# thread_pool.vectortile.queue_size: 4
+
+
+# threadpool.index.type: fixed
+# threadpool.index.size: 64
+# threadpool.index.queue_size: 1000
+# indices.memory.index_buffer_size: 20%
+
+# threadpool.search.size: 64
+# threadpool.search.type: fixed
+# threadpool.search.queue_size: 1000
+# threadpool.get.type: fixed
+# threadpool.get.size: 32
+# threadpool.get.queue_size: 1000
+# threadpool.bulk.size: 32
+# threadpool.bulk.queue_size: 1000
+# threadpool.flush.type: fixed
+# threadpool.flush.size: 32
+# threadpool.flush.queue_size: 1000
+
+# indices.store.throttle.max_bytes_per_sec: 100MB
+
+
+# index.analysis.analyzer.ik.type: "ik"
+# index.merge.scheduler.max_thread_count: 8
+# index.translog.flush_threshold_size: 200MB
+# index.translog.durability: async
+# index.translog.flush_threshold_ops: 500000
+
+# node.max_local_storage_nodes: 2
+# discovery.zen.ping.unicast.hosts: ["192.168.0.100"]
+# discovery.zen.minimum_master_nodes: 1
+# discovery.zen.ping.multicast.enabled: false
+# discovery.zen.ping.multicast.group: 224.2.2.4
+# discovery.zen.ping.multicast.port : 54328
+# discovery.zen.ping.multicast.ttl: 3
+# discovery.zen.ping.multicast.address: 0.0.0.0
+# node.master: true
+# node.data: true
+# index.number_of_shards: 5
+# index.number_of_replicas: 2
+#indices.fielddata.cache.size: 50%
+#
+http.cors.enabled: true
+http.cors.allow-origin: "*"
+http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE
+http.cors.allow-headers : Authorization, X-Requested-With,X-Auth-Token,Content-Type, Content-Length
+
+
+transport.tcp.port: 9300
+# 设置节点之间交互的tcp端口，默认是9300。
+# transport.tcp.compress: true
+# 设置是否压缩tcp传输时的数据，默认为false，不压缩。
+
+# reindex.remote.whitelist: "192.168.0.100:9300"
+
+# https://docs.zammad.org/en/latest/install/elasticsearch.html#step-2-suggested-configuration
+# Performance may suffer if it is set too high.
+http.max_content_length: 400mb
+
+# Allows the engine to generate larger (more complex) search queries.
+# Elasticsearch will raise an error or deprecation notice if this value is too low,
+# but setting it too high can overload system resources (Default: 1024).
+#
+# Available in version 6.6+ only.
+indices.query.bool.max_clause_count: 20000
+# indexing_pressure.memory.limit: 30
+cluster.routing.allocation.disk.threshold_enabled: false
+# 开启就无法启动
+# bootstrap.memory_lock: true

config/jvm.options

@@ -0,0 +1,21 @@
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+-Djava.io.tmpdir=${ES_TMPDIR}
+-XX:+HeapDumpOnOutOfMemoryError
+9-:-XX:+ExitOnOutOfMemoryError
+-XX:HeapDumpPath=data
+-XX:ErrorFile=logs/hs_err_pid%p.log
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:logs/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+
+-Xms4g
+-Xmx4g

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"embed"
 	"github.com/hktalent/scan4all/pkg"
 	"github.com/hktalent/scan4all/pkg/hydra"
 	naaburunner "github.com/hktalent/scan4all/pkg/naabu/v2/pkg/runner"
@@ -9,6 +10,9 @@ import (
 	"runtime"
 )
 
+//go:embed config/*
+var config embed.FS
+
 func main() {
 	defer func() {
 		pkg.Cache1.Close()

nuclei_Yaml/nuclei_yaml.go

@@ -68,7 +68,6 @@ func RunNuclei(buf bytes.Buffer, xx chan bool) {
 	}
 	nucleiRunner.Close()
 }
-
 func readConfig() {
 	options.Targets = []string{}
 	options.TargetsFilePath = ""
@@ -111,7 +110,12 @@ func readConfig() {
 	options.FollowRedirects = false
 	options.MaxRedirects = 10
 	options.DisableRedirects = false
+
 	options.ReportingConfig = ""
+	// 启动es记录
+	if "true" == pkg.GetVal("enableEsSv") {
+		options.ReportingConfig = "config/nuclei_esConfig.yaml"
+	}
 	options.CustomHeaders = []string{}
 	options.Vars = goflags.RuntimeMap{}
 	options.ResolversFile = ""
@@ -186,7 +190,7 @@ func readConfig() {
 	//	flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"),
 	//	flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
 	//	flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable redirects for http templates"),
-	//	flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
+	//	flagSet.StringVarP(&options.ReportingConfigReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking
 	//	flagSet.FileStringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom header/cookie to include in all http request in header:value format (cli, file)"),
 	//	flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in key=value format"),
 	//	flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"),
@@ -289,8 +293,13 @@ func readConfig() {
 	options.UpdateNuclei = false
 	options.UpdateTemplates = false
 	// 嵌入式集成私人版本nuclei-templates 共3744个YAML POC
-	options.TemplatesDirectory = "config/nuclei-templates"
-	options.NoUpdateTemplates = true
+	if "true" == pkg.GetVal("enablEmbedYaml") {
+		options.TemplatesDirectory = "config/nuclei-templates"
+		options.NoUpdateTemplates = true
+	} else {
+		options.TemplatesDirectory = ""
+		options.NoUpdateTemplates = false
+	}
 	options.EnableProgressBar = false
 	options.StatsJSON = false
 	options.StatsInterval = 5

pkg/config.go

@@ -36,6 +36,13 @@ func GetVal(key string) string {
 	}
 	return os.Getenv(key)
 }
+func GetValByDefault(key, dftvl string) string {
+	s := GetVal(key)
+	if "" == s {
+		return dftvl
+	}
+	return s
+}
 
 var (
 	Naabu = "naabu"

pkg/domain.go

@@ -34,6 +34,7 @@ func doAppends(a []string, s []string) []string {
 }
 
 func doSub(s string) (aRst []string, err1 error) {
+	bSend := false
 	if "*." == s[:2] {
 		EnableSubfinder := GetVal(EnableSubfinder)
 		if "" != EnableSubfinder {
@@ -52,10 +53,14 @@ func doSub(s string) (aRst []string, err1 error) {
 					}
 				}
 			}
+			bSend = true
 		} else {
 			aRst = append(aRst, s[2:])
 		}
 	}
+	if bSend {
+		go SendAData[string](s[:2], aRst, "subfinder")
+	}
 	return aRst, nil
 }
 

pkg/hydra/doNmapResult.go

@@ -34,21 +34,38 @@ func DoParseXml(s string) {
 		log.Println(err)
 		return
 	}
+	var enableEsSv = pkg.GetVal("enableEsSv")
+	m1 := make(map[string][][]string)
 	for _, n := range xmlquery.Find(doc, "//host") {
 		x1 := n.SelectElement("address").Attr[0].Value
 		ps := n.SelectElements("ports/port")
 		for _, x := range ps {
 			if "open" == x.SelectElement("state").Attr[0].Value {
 				ip := x1
-				port, _ := strconv.Atoi(GetAttr(x.Attr, "portid"))
+				szPort := GetAttr(x.Attr, "portid")
+				port, _ := strconv.Atoi(szPort)
 				service := GetAttr(x.SelectElement("service").Attr, "name")
 				go CheckWeakPassword(ip, service, port)
 				// 存储结果到其他地方
 				//x9 := AuthInfo{IPAddr: ip, Port: port, Protocol: service}
+				if "true" == enableEsSv {
+					var xx09 = [][]string{}
+					if a1, ok := m1[ip]; ok {
+						xx09 = a1
+					}
+					m1[ip] = append(xx09, []string{szPort, service})
+				}
 				//fmt.Printf("%s\t%d\t%s\n", ip, port, service)
 			}
 		}
 	}
+	if "true" == enableEsSv {
+		if 0 < len(m1) {
+			for k, x := range m1 {
+				pkg.SendAData[[]string](k, x, "nmap")
+			}
+		}
+	}
 }
 
 func DoNmapRst() {