Skip to content

Commit e81a98a

Browse files
committed
add network hash
1 parent 3f43da4 commit e81a98a

File tree

3 files changed

+247
-1
lines changed

3 files changed

+247
-1
lines changed

docker/helk-logstash/output_templates/99-logs-any-fields.json

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"order": 99,
33
"index_patterns": [ "logs-*" ],
4-
"version": 2018080101,
4+
"version": 2019060301,
55
"mappings": {
66
"properties": {
77
"any_ip_addr": {
@@ -19,13 +19,24 @@
1919
}
2020
}
2121
},
22+
"fingerprint_network_community_id": {
23+
"type": "keyword"
24+
},
2225
"related": {
2326
"properties": {
2427
"ip": {
2528
"type": "alias",
2629
"path": "any_ip_addr"
2730
}
2831
}
32+
},
33+
"network": {
34+
"properties": {
35+
"community_id": {
36+
"type": "alias",
37+
"path": "fingerprint_network_community_id"
38+
}
39+
}
2940
}
3041
}
3142
}
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# HELK community-id filter conf
2+
# HELK build Stage: Alpha
3+
# Author: Nate Guagenti (@neu5ron)
4+
# License: GPL-3.0
5+
6+
filter {
7+
# Lookup community id event's containing network parameters
8+
if [src_ip_addr] and [dst_ip_addr] and [network_protocol] and [dst_port] and [src_port] and [@metadata][src_ip_addr][number_of_ip_addresses] == 1 and [@metadata][dst_ip_addr][number_of_ip_addresses] == 1 {
9+
ruby {
10+
path => "/usr/share/logstash/pipeline/ruby/community-id.rb"
11+
script_params => {
12+
"source_ip_field" => "src_ip_addr"
13+
"dest_ip_field" => "dst_ip_addr"
14+
"source_port_field" => "src_port"
15+
"dest_port_field" => "dst_port"
16+
"protocol_field" => "network_protocol"
17+
"target_field" => "fingerprint_network_community_id"
18+
}
19+
tag_on_exception => "_rubyexception-community_id"
20+
}
21+
}
22+
}
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
require 'socket'
2+
require 'digest'
3+
require 'base64'
4+
5+
TRANSPORT_PROTOS = ['icmp', 'icmp6', 'tcp', 'udp', 'sctp']
6+
7+
PROTO_MAP = {
8+
'icmp' => 1,
9+
'tcp' => 6,
10+
'udp' => 17,
11+
'icmp6' => 58
12+
}
13+
14+
ICMP4_MAP = {
15+
# Echo => Reply
16+
8 => 0,
17+
# Reply => Echo
18+
0 => 8,
19+
# Timestamp => TS reply
20+
13 => 14,
21+
# TS reply => timestamp
22+
14 => 13,
23+
# Info request => Info Reply
24+
15 => 16,
25+
# Info Reply => Info Req
26+
16 => 15,
27+
# Rtr solicitation => Rtr Adverstisement
28+
10 => 9,
29+
# Mask => Mask reply
30+
17 => 18,
31+
# Mask reply => Mask
32+
18 => 17,
33+
}
34+
35+
ICMP6_MAP = {
36+
# Echo Request => Reply
37+
128 => 129,
38+
# Echo Reply => Request
39+
129 => 128,
40+
# Router Solicit => Advert
41+
133 => 134,
42+
# Router Advert => Solicit
43+
134 => 133,
44+
# Neighbor Solicit => Advert
45+
135 => 136,
46+
# Neighbor Advert => Solicit
47+
136 => 135,
48+
# Multicast Listener Query => Report
49+
130 => 131,
50+
# Multicast Report => Listener Query
51+
131 => 130,
52+
# Node Information Query => Response
53+
139 => 140,
54+
# Node Information Response => Query
55+
140 => 139,
56+
# Home Agent Address Discovery Request => Reply
57+
144 => 145,
58+
# Home Agent Address Discovery Reply => Request
59+
145 => 144,
60+
}
61+
62+
VERSION = '1:'
63+
64+
def bin_to_hex(s)
65+
s.each_byte.map { |b| b.to_s(16).rjust(2, '0') }.join(':')
66+
end
67+
68+
def register(params)
69+
@use_base64 = params.fetch("use_base64", "true")
70+
@comm_id_seed = params.fetch("community_id_seed", "0").to_i
71+
@target_field = params["target_field"]
72+
@source_ip = params["source_ip_field"]
73+
@source_port = params["source_port_field"]
74+
@dest_ip = params["dest_ip_field"]
75+
@dest_port = params["dest_port_field"]
76+
@protocol = params["protocol_field"]
77+
end
78+
79+
def filter(event)
80+
81+
if @target_field.nil?
82+
event.tag("community_id_target_field_not_set")
83+
return [event]
84+
end
85+
86+
# Tag and quit if any fields aren't present
87+
[@source_ip, @source_port, @dest_ip, @dest_port, @protocol].each do |field|
88+
if event.get(field).nil?
89+
event.tag("#{field}_not_found")
90+
return [event]
91+
end
92+
end
93+
94+
# Retreive the fields
95+
src_ip = event.get("#{@source_ip}")
96+
src_p = event.get("#{@source_port}").to_i
97+
dst_ip = event.get("#{@dest_ip}")
98+
dst_p = event.get("#{@dest_port}").to_i
99+
protocol = event.get("#{@protocol}")
100+
101+
# Parse to sockaddr_in struct bytestring
102+
src = Socket.sockaddr_in(src_p, src_ip)
103+
dst = Socket.sockaddr_in(dst_p, dst_ip)
104+
105+
is_one_way = false
106+
# Special case handling for ICMP type/codes
107+
if protocol == 'icmp' || protocol == 'icmp6'
108+
if src.length == 16 # IPv4
109+
if ICMP4_MAP.has_key?(src_p) == false
110+
is_one_way = true
111+
end
112+
elsif src.length == 28 # IPv6
113+
if ICMP6_MAP.has_key?(src_p) == false
114+
is_one_way = true
115+
end
116+
# Set this correctly if not already set
117+
protocol = 'icmp6'
118+
end
119+
end
120+
121+
# Fetch the protocol number
122+
proto = PROTO_MAP.fetch(protocol.downcase, 0)
123+
124+
# Parse out the network-ordered bytestrings for ip/ports....####zDamTyILGeKD4H0####
125+
if src.length == 16 # IPv4
126+
sip = src[4,4]
127+
sport = src[2,2]
128+
elsif src.length == 28 # IPv6
129+
sip = src[4,16]
130+
sport = src[2,2]
131+
end
132+
if dst.length == 16 # IPv4
133+
dip = dst[4,4]
134+
dport = dst[2,2]
135+
elsif dst.length == 28 # IPv6
136+
dip = dst[4,16]
137+
dport = dst[2,2]
138+
end
139+
140+
if !(is_one_way || ((sip <=> dip) == -1) || ((sip == dip) && ((sport <=> dport) < 1))
141+
mip = sip
142+
mport = sport
143+
sip = dip
144+
sport = dport
145+
dip = mip
146+
dport = mport
147+
end
148+
149+
# Hash all the things
150+
hash = Digest::SHA1.new
151+
hash.update([@comm_id_seed].pack('n')) # 2-byte seed
152+
153+
hash.update(sip) # 4 bytes (v4 addr) or 16 bytes (v6 addr)
154+
hash.update(dip) # 4 bytes (v4 addr) or 16 bytes (v6 addr)####IbPK6g####
155+
156+
hash.update([proto].pack('C')) # 1 byte for transport proto
157+
hash.update([0].pack('C')) # 1 byte padding
158+
159+
# If transport protocol, hash the ports too
160+
hash.update(sport) # 2 bytes for port
161+
hash.update(dport) # 2 bytes for port
162+
163+
comm_id = nil
164+
165+
if @use_base64
166+
comm_id = VERSION + Base64.strict_encode64(hash.digest)
167+
else
168+
comm_id = VERSION + hash.hexdigest
169+
end
170+
171+
172+
event.set("#{@target_field}", comm_id)
173+
174+
return [event]
175+
end
176+
177+
### Validation Tests
178+
179+
test "when proto is tcpv4" do
180+
parameters {{"source_ip_field" => "src_ip", "dest_ip_field" => "dst_ip", "source_port_field" => "src_port", "dest_port_field" => "dst_port", "protocol_field" => "protocol", "target_field" => "community_id" }}
181+
in_event {{ "dst_ip" => "66.35.250.204", "src_ip" => "128.232.110.120", "dst_port" => 80, "src_port" => 34855, "protocol" => "tcp" }}
182+
expect("the hash is computed") {|events| events.first.get("community_id") == "1:LQU9qZlK+B5F3KDmev6m5PMibrg=" }
183+
end
184+
185+
test "when proto is udpv4" do
186+
parameters {{"source_ip_field" => "src_ip", "dest_ip_field" => "dst_ip", "source_port_field" => "src_port", "dest_port_field" => "dst_port", "protocol_field" => "protocol", "target_field" => "community_id" }}
187+
in_event {{ "dst_ip" => "8.8.8.8", "src_ip" => "192.168.1.52", "dst_port" => 53, "src_port" => 54585, "protocol" => "udp" }}
188+
expect("the hash is computed") {|events| events.first.get("community_id") == "1:d/FP5EW3wiY1vCndhwleRRKHowQ=" }
189+
end
190+
191+
test "when proto is IPv6" do
192+
parameters {{"source_ip_field" => "src_ip", "dest_ip_field" => "dst_ip", "source_port_field" => "src_port", "dest_port_field" => "dst_port", "protocol_field" => "protocol", "target_field" => "community_id" }}
193+
in_event {{ "dst_ip" => "2607:f8b0:400c:c03::1a", "src_ip" => "2001:470:e5bf:dead:4957:2174:e82c:4887", "dst_port" => 25, "src_port" => 63943, "protocol" => "tcp" }}
194+
expect("the hash is computed") {|events| events.first.get("community_id") == "1:/qFaeAR+gFe1KYjMzVDsMv+wgU4=" }
195+
end
196+
197+
test "when proto is icmpv4" do
198+
parameters {{"source_ip_field" => "src_ip", "dest_ip_field" => "dst_ip", "source_port_field" => "src_port", "dest_port_field" => "dst_port", "protocol_field" => "protocol", "target_field" => "community_id" }}
199+
in_event {{ "dst_ip" => "192.168.0.1", "src_ip" => "192.168.0.89", "dst_port" => 0, "src_port" => 8, "protocol" => "icmp" }}
200+
expect("the hash is computed") {|events| events.first.get("community_id") == "1:X0snYXpgwiv9TZtqg64sgzUn6Dk=" }
201+
end
202+
203+
test "when proto is icmpv6" do
204+
parameters {{"source_ip_field" => "src_ip", "dest_ip_field" => "dst_ip", "source_port_field" => "src_port", "dest_port_field" => "dst_port", "protocol_field" => "protocol", "target_field" => "community_id" }}
205+
in_event {{ "dst_ip" => "3ffe:507:0:1:200:86ff:fe05:80da", "src_ip" => "3ffe:501:0:1802:260:97ff:feb6:7ff0", "dst_port" => 0, "src_port" => 3, "protocol" => "icmp" }}
206+
expect("the hash is computed") {|events| events.first.get("community_id") == "1:bnQKq8A2r//dWnkRW2EYcMhShjc=" }
207+
end
208+
209+
test "when field doesn't exist" do
210+
parameters { {"source_ip_field" => "src_ip", "dest_ip_field" => "dst_ip", "source_port_field" => "src_port", "dest_port_field" => "dst_port", "protocol_field" => "protocol", "target_field" => "community_id" } }
211+
in_event {{ "dst_ip" => "8.8.8.8", "source_ip" => "192.168.1.52", "dst_port" => 53, "src_port" => 54585, "protocol" => "udp" }}
212+
expect("tags as not found") {|events| events.first.get("tags").include?("src_ip_not_found") }
213+
end

0 commit comments

Comments
 (0)