Skip to content

Commit 6bc4651

Browse files
AryanBakliwalAryanBakliwal
authored
Add bidirectional support to PacketCapture (#6882)
A new `direction` field is added to the PacketCapture CRD. Packets can now be captured from source to destination (default), destination to source, or in both directions. Fixes #6862 Signed-off-by: Aryan Bakliwal <[email protected]>
1 parent 5301249 commit 6bc4651

17 files changed

+488
-95
lines changed

‎build/charts/antrea/crds/packetcapture.yaml

+4-1
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,10 @@ spec:
152152
type: integer
153153
minimum: 1
154154
maximum: 65535
155-
155+
direction:
156+
type: string
157+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
158+
default: "SourceToDestination"
156159
timeout:
157160
type: integer
158161
minimum: 1

‎build/yamls/antrea-aks.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -3067,7 +3067,10 @@ spec:
30673067
type: integer
30683068
minimum: 1
30693069
maximum: 65535
3070-
3070+
direction:
3071+
type: string
3072+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
3073+
default: "SourceToDestination"
30713074
timeout:
30723075
type: integer
30733076
minimum: 1

‎build/yamls/antrea-crds.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -3040,7 +3040,10 @@ spec:
30403040
type: integer
30413041
minimum: 1
30423042
maximum: 65535
3043-
3043+
direction:
3044+
type: string
3045+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
3046+
default: "SourceToDestination"
30443047
timeout:
30453048
type: integer
30463049
minimum: 1

‎build/yamls/antrea-eks.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -3067,7 +3067,10 @@ spec:
30673067
type: integer
30683068
minimum: 1
30693069
maximum: 65535
3070-
3070+
direction:
3071+
type: string
3072+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
3073+
default: "SourceToDestination"
30713074
timeout:
30723075
type: integer
30733076
minimum: 1

‎build/yamls/antrea-gke.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -3067,7 +3067,10 @@ spec:
30673067
type: integer
30683068
minimum: 1
30693069
maximum: 65535
3070-
3070+
direction:
3071+
type: string
3072+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
3073+
default: "SourceToDestination"
30713074
timeout:
30723075
type: integer
30733076
minimum: 1

‎build/yamls/antrea-ipsec.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -3067,7 +3067,10 @@ spec:
30673067
type: integer
30683068
minimum: 1
30693069
maximum: 65535
3070-
3070+
direction:
3071+
type: string
3072+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
3073+
default: "SourceToDestination"
30713074
timeout:
30723075
type: integer
30733076
minimum: 1

‎build/yamls/antrea.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -3067,7 +3067,10 @@ spec:
30673067
type: integer
30683068
minimum: 1
30693069
maximum: 65535
3070-
3070+
direction:
3071+
type: string
3072+
enum: ["SourceToDestination", "DestinationToSource", "Both"]
3073+
default: "SourceToDestination"
30713074
timeout:
30723075
type: integer
30733076
minimum: 1

‎docs/packetcapture-guide.md

+3
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ the target traffic flow:
3232
* Destination Pod, or IP address
3333
* Transport protocol (TCP/UDP/ICMP)
3434
* Transport ports
35+
* Direction (SourceToDestination/DestinationToSource/Both)
3536
3637
You can start a new packet capture by creating a `PacketCapture` CR. An optional `fileServer`
3738
field can be specified to store the generated packets file. Before that,
@@ -74,6 +75,8 @@ spec:
7475
pod:
7576
namespace: default
7677
name: backend
78+
# Available options for direction: `SourceToDestination` (default), `DestinationToSource` or `Both`.
79+
direction: SourceToDestination # optional to specify
7780
packet:
7881
ipFamily: IPv4
7982
protocol: TCP # support arbitrary number values and string values in [TCP,UDP,ICMP] (case insensitive)

‎pkg/agent/packetcapture/capture/bpf.go

+181-51
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,76 @@ func compareProtocol(protocol uint32, skipTrue, skipFalse uint8) bpf.Instruction
7272
return bpf.JumpIf{Cond: bpf.JumpEqual, Val: protocol, SkipTrue: skipTrue, SkipFalse: skipFalse}
7373
}
7474

75+
func calculateSkipFalse(srcPort, dstPort uint16) uint8 {
76+
var count uint8
77+
// load dstIP and compare
78+
count += 2
79+
80+
if srcPort > 0 || dstPort > 0 {
81+
// load fragment offset
82+
count += 3
83+
84+
if srcPort > 0 {
85+
count += 2
86+
}
87+
if dstPort > 0 {
88+
count += 2
89+
}
90+
}
91+
// ret keep
92+
count += 1
93+
94+
return count
95+
}
96+
97+
// Generates IP address and port matching instructions
98+
func compileIPPortFilter(srcAddrVal, dstAddrVal uint32, size, curLen uint8, srcPort, dstPort uint16, needsOtherTrafficDirectionCheck bool) []bpf.Instruction {
99+
inst := []bpf.Instruction{}
100+
101+
// from here we need to check the inst length to calculate skipFalse. If no protocol is set, there will be no related bpf instructions.
102+
103+
// calculate skip size to jump to the final instruction (NO MATCH)
104+
skipToEnd := func() uint8 {
105+
return size - curLen - uint8(len(inst)) - 2
106+
}
107+
108+
// needsOtherTrafficDirectionCheck indicates if we need to check whether the packet belongs to the return traffic flow when source IP from the
109+
// packet spec and packet header don't match and we are capturing packets in both direction. If true, we calculate skipFalse to jump to the
110+
// instruction that compares the destination IP from the packet spec with the loaded source IP from the packet header.
111+
if needsOtherTrafficDirectionCheck {
112+
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: srcAddrVal, SkipTrue: 0, SkipFalse: calculateSkipFalse(srcPort, dstPort)})
113+
} else {
114+
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: srcAddrVal, SkipTrue: 0, SkipFalse: skipToEnd()})
115+
}
116+
117+
// dst ip
118+
inst = append(inst, loadIPv4DestinationAddress)
119+
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: dstAddrVal, SkipTrue: 0, SkipFalse: skipToEnd()})
120+
121+
if srcPort > 0 || dstPort > 0 {
122+
skipTrue := skipToEnd() - 1
123+
inst = append(inst, loadIPv4HeaderOffset(skipTrue)...)
124+
if srcPort > 0 {
125+
inst = append(inst, loadIPv4SourcePort)
126+
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(srcPort), SkipTrue: 0, SkipFalse: skipToEnd()})
127+
}
128+
if dstPort > 0 {
129+
inst = append(inst, loadIPv4DestinationPort)
130+
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(dstPort), SkipTrue: 0, SkipFalse: skipToEnd()})
131+
}
132+
}
133+
134+
// return (accept)
135+
inst = append(inst, returnKeep)
136+
137+
return inst
138+
}
139+
75140
// compilePacketFilter compiles the CRD spec to bpf instructions. For now, we only focus on
76141
// ipv4 traffic. Compared to the raw BPF filter supported by libpcap, we only need to support
77142
// limited use cases, so an expression parser is not needed.
78-
func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) []bpf.Instruction {
79-
size := uint8(calculateInstructionsSize(packetSpec))
143+
func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP, direction crdv1alpha1.CaptureDirection) []bpf.Instruction {
144+
size := uint8(calculateInstructionsSize(packetSpec, direction))
80145

81146
// ipv4 check
82147
inst := []bpf.Instruction{loadEtherKind}
@@ -101,20 +166,8 @@ func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) []
101166
}
102167
}
103168

104-
// source ip
105-
if srcIP != nil {
106-
inst = append(inst, loadIPv4SourceAddress)
107-
addrVal := binary.BigEndian.Uint32(srcIP[len(srcIP)-4:])
108-
// from here we need to check the inst length to calculate skipFalse. If no protocol is set, there will be no related bpf instructions.
109-
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: addrVal, SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2})
110-
111-
}
112-
// dst ip
113-
if dstIP != nil {
114-
inst = append(inst, loadIPv4DestinationAddress)
115-
addrVal := binary.BigEndian.Uint32(dstIP[len(dstIP)-4:])
116-
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: addrVal, SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2})
117-
}
169+
srcAddrVal := binary.BigEndian.Uint32(srcIP[len(srcIP)-4:])
170+
dstAddrVal := binary.BigEndian.Uint32(dstIP[len(dstIP)-4:])
118171

119172
// ports
120173
var srcPort, dstPort uint16
@@ -134,22 +187,18 @@ func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) []
134187
}
135188
}
136189

137-
if srcPort > 0 || dstPort > 0 {
138-
skipTrue := size - uint8(len(inst)) - 3
139-
inst = append(inst, loadIPv4HeaderOffset(skipTrue)...)
140-
if srcPort > 0 {
141-
inst = append(inst, loadIPv4SourcePort)
142-
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(srcPort), SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2})
143-
}
144-
if dstPort > 0 {
145-
inst = append(inst, loadIPv4DestinationPort)
146-
inst = append(inst, bpf.JumpIf{Cond: bpf.JumpEqual, Val: uint32(dstPort), SkipTrue: 0, SkipFalse: size - uint8(len(inst)) - 2})
147-
}
190+
inst = append(inst, loadIPv4SourceAddress)
148191

192+
if direction == crdv1alpha1.CaptureDirectionSourceToDestination {
193+
inst = append(inst, compileIPPortFilter(srcAddrVal, dstAddrVal, size, uint8(len(inst)), srcPort, dstPort, false)...)
194+
} else if direction == crdv1alpha1.CaptureDirectionDestinationToSource {
195+
inst = append(inst, compileIPPortFilter(dstAddrVal, srcAddrVal, size, uint8(len(inst)), dstPort, srcPort, false)...)
196+
} else {
197+
inst = append(inst, compileIPPortFilter(srcAddrVal, dstAddrVal, size, uint8(len(inst)), srcPort, dstPort, true)...)
198+
inst = append(inst, compileIPPortFilter(dstAddrVal, srcAddrVal, size, uint8(len(inst)), dstPort, srcPort, false)...)
149199
}
150200

151-
// return
152-
inst = append(inst, returnKeep)
201+
// return (drop)
153202
inst = append(inst, returnDrop)
154203

155204
return inst
@@ -169,50 +218,131 @@ func compilePacketFilter(packetSpec *crdv1alpha1.Packet, srcIP, dstIP net.IP) []
169218
// (006) ld [30] # Load 4B at 30 (dest address)
170219
// (007) jeq #0x7f000001 jt 8 jf 16 # If bytes match(127.0.0.1), goto #8, else #16
171220
// (008) ldh [20] # Load 2B at 20 (13b Fragment Offset)
172-
// (009) jset #0x1fff jt 16 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #16
221+
// (009) jset #0x1fff jt 16 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #16
173222
// (010) ldxb 4*([14]&0xf) # x = IP header length
174223
// (011) ldh [x + 14] # Load 2B at x+14 (TCP Source Port)
175-
// (012) jeq #0x7b jt 13 jf 16 # TCP Source Port: If 123, goto #13, else #16
224+
// (012) jeq #0x7b jt 13 jf 16 # TCP Source Port: If 123, goto #13, else #16
176225
// (013) ldh [x + 16] # Load 2B at x+16 (TCP dst port)
177-
// (014) jeq #0x7c jt 15 jf 16 # TCP dst port: If 123, goto $15, else #16
226+
// (014) jeq #0x7c jt 15 jf 16 # TCP dst port: If 123, goto #15, else #16
178227
// (015) ret #262144 # MATCH
179228
// (016) ret #0 # NOMATCH
180229

181-
func calculateInstructionsSize(packet *crdv1alpha1.Packet) int {
230+
// When capturing return traffic also (i.e., both src -> dst and dst -> src), the filter might look like this:
231+
// 'ip proto 6 and ((src host 10.244.1.2 and dst host 10.244.1.3 and src port 123 and dst port 124) or (src host 10.244.1.3 and dst host 10.244.1.2 and src port 124 and dst port 123))'
232+
// And using `tcpdump -i <device> '<filter>' -d` will generate the following BPF instructions:
233+
// (000) ldh [12] # Load 2B at 12 (Ethertype)
234+
// (001) jeq #0x800 jt 2 jf 26 # Ethertype: If IPv4, goto #2, else #26
235+
// (002) ldb [23] # Load 1B at 23 (IPv4 Protocol)
236+
// (003) jeq #0x6 jt 4 jf 26 # IPv4 Protocol: If TCP, goto #4, #26
237+
// (004) ld [26] # Load 4B at 26 (source address)
238+
// (005) jeq #0xaf40102 jt 6 jf 15 # If bytes match(10.244.1.2), goto #6, else #15
239+
// (006) ld [30] # Load 4B at 30 (dest address)
240+
// (007) jeq #0xaf40103 jt 8 jf 26 # If bytes match(10.244.1.3), goto #8, else #26
241+
// (008) ldh [20] # Load 2B at 20 (13b Fragment Offset)
242+
// (009) jset #0x1fff jt 26 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #26
243+
// (010) ldxb 4*([14]&0xf) # x = IP header length
244+
// (011) ldh [x + 14] # Load 2B at x+14 (TCP Source Port)
245+
// (012) jeq #0x7b jt 13 jf 26 # TCP Source Port: If 123, goto #13, else #26
246+
// (013) ldh [x + 16] # Load 2B at x+16 (TCP dst port)
247+
// (014) jeq #0x7c jt 25 jf 26 # TCP dst port: If 123, goto #25, else #26
248+
// (015) jeq #0xaf40103 jt 16 jf 26 # If bytes match(10.244.1.3), goto #16, else #26
249+
// (016) ld [30] # Load 4B at 30 (return traffic dest address)
250+
// (017) jeq #0xaf40102 jt 18 jf 26 # If bytes match(10.244.1.2), goto #18, else #26
251+
// (018) ldh [20] # Load 2B at 20 (13b Fragment Offset)
252+
// (019) jset #0x1fff jt 26 jf 20 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #20, else #26
253+
// (020) ldxb 4*([14]&0xf) # x = IP header length
254+
// (021) ldh [x + 14] # Load 2B at x+14 (TCP Source Port)
255+
// (022) jeq #0x7c jt 23 jf 26 # TCP Source Port: If 124, goto #23, else #26
256+
// (023) ldh [x + 16] # Load 2B at x+16 (TCP dst port)
257+
// (024) jeq #0x7b jt 25 jf 26 # TCP dst port: If 123, goto #25, else #26
258+
// (025) ret #262144 # MATCH
259+
// (026) ret #0 # NOMATCH
260+
261+
// For simpler code generation in 'Both' direction, an extra instruction to accept the packet is added after instruction 014.
262+
// The final instruction set looks like this:
263+
// (000) ldh [12] # Load 2B at 12 (Ethertype)
264+
// (001) jeq #0x800 jt 2 jf 27 # Ethertype: If IPv4, goto #2, else #27
265+
// (002) ldb [23] # Load 1B at 23 (IPv4 Protocol)
266+
// (003) jeq #0x6 jt 4 jf 27 # IPv4 Protocol: If TCP, goto #4, #27
267+
// (004) ld [26] # Load 4B at 26 (source address)
268+
// (005) jeq #0xaf40102 jt 6 jf 16 # If bytes match(10.244.1.2), goto #6, else #16
269+
// (006) ld [30] # Load 4B at 30 (dest address)
270+
// (007) jeq #0xaf40103 jt 8 jf 27 # If bytes match(10.244.1.3), goto #8, else #27
271+
// (008) ldh [20] # Load 2B at 20 (13b Fragment Offset)
272+
// (009) jset #0x1fff jt 27 jf 10 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #10, else #27
273+
// (010) ldxb 4*([14]&0xf) # x = IP header length
274+
// (011) ldh [x + 14] # Load 2B at x+14 (TCP Source Port)
275+
// (012) jeq #0x7b jt 13 jf 27 # TCP Source Port: If 123, goto #13, else #27
276+
// (013) ldh [x + 16] # Load 2B at x+16 (TCP dst port)
277+
// (014) jeq #0x7c jt 15 jf 27 # TCP dst port: If 123, goto #15, else #27
278+
// (015) ret #262144 # MATCH
279+
// (016) jeq #0xaf40103 jt 17 jf 27 # If bytes match(10.244.1.3), goto #17, else #27
280+
// (017) ld [30] # Load 4B at 30 (return traffic dest address)
281+
// (018) jeq #0xaf40102 jt 19 jf 27 # If bytes match(10.244.1.2), goto #19, else #27
282+
// (019) ldh [20] # Load 2B at 20 (13b Fragment Offset)
283+
// (020) jset #0x1fff jt 27 jf 21 # Use 0x1fff as a mask for fragment offset; If fragment offset != 0, #21, else #27
284+
// (021) ldxb 4*([14]&0xf) # x = IP header length
285+
// (022) ldh [x + 14] # Load 2B at x+14 (TCP Source Port)
286+
// (023) jeq #0x7c jt 24 jf 27 # TCP Source Port: If 124, goto #24, else #27
287+
// (024) ldh [x + 16] # Load 2B at x+16 (TCP dst port)
288+
// (025) jeq #0x7b jt 26 jf 27 # TCP dst port: If 123, goto #26, else #27
289+
// (026) ret #262144 # MATCH
290+
// (027) ret #0 # NOMATCH
291+
292+
func calculateInstructionsSize(packet *crdv1alpha1.Packet, direction crdv1alpha1.CaptureDirection) int {
182293
count := 0
183294
// load ethertype
184295
count++
185296
// ip check
186297
count++
187298

299+
// src and dst ip
300+
count += 4
301+
188302
if packet != nil {
189303
// protocol check
190304
if packet.Protocol != nil {
191305
count += 2
192306
}
193-
transPort := packet.TransportHeader
194-
if transPort.TCP != nil {
195-
// load Fragment Offset
196-
count += 3
197-
if transPort.TCP.SrcPort != nil {
198-
count += 2
199-
}
200-
if transPort.TCP.DstPort != nil {
201-
count += 2
307+
transport := packet.TransportHeader
308+
portFiltersSize := func() int {
309+
count := 0
310+
if transport.TCP != nil {
311+
// load Fragment Offset
312+
count += 3
313+
if transport.TCP.SrcPort != nil {
314+
count += 2
315+
}
316+
if transport.TCP.DstPort != nil {
317+
count += 2
318+
}
319+
320+
} else if transport.UDP != nil {
321+
count += 3
322+
if transport.UDP.SrcPort != nil {
323+
count += 2
324+
}
325+
if transport.UDP.DstPort != nil {
326+
count += 2
327+
}
202328
}
329+
return count
330+
}()
331+
332+
count += portFiltersSize
203333

204-
} else if transPort.UDP != nil {
334+
if direction == crdv1alpha1.CaptureDirectionBoth {
335+
336+
// extra returnKeep
337+
count++
338+
339+
// src and dst ip (return traffic)
205340
count += 3
206-
if transPort.UDP.SrcPort != nil {
207-
count += 2
208-
}
209-
if transPort.UDP.DstPort != nil {
210-
count += 2
211-
}
341+
342+
count += portFiltersSize
343+
212344
}
213345
}
214-
// src and dst ip
215-
count += 4
216346

217347
// ret command
218348
count += 2

0 commit comments

Comments
 (0)