Skip to content

Commit 6d0b142

Browse files
spencercjhspencercjh
authored
feat: check cap privileges instead of Geteuid during starting the agent (#242)
* feat: Introduce github.com/containerd/containerd/pkg/cap to check whether process has CAP_BPF privilege Signed-off-by: spencercjh <[email protected]> * fix: better logs * fix: adapt to e2e test env * style: go mod tidy * fix: make tests pass * fix: DO NOT use containerd cap package * test: introduce tests to verify agent/common/permission.go * fix: correct implementation refer to https://man7.org/linux/man-pages/man2/capset.2.html * test: test test_add_cap_bpf first * test: cap-add difference capability for different kernal * test: load btf file to container and run kyanos with --btf flag * test: add missing capability CAP_SYS_RESOURCE * test: try to use --privileged instead of cap-add --------- Signed-off-by: spencercjh <[email protected]>
1 parent ca70f6d commit 6d0b142

File tree

8 files changed

+137
-33
lines changed

8 files changed

+137
-33
lines changed

‎.github/workflows/test.yml

+55-28
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,33 @@ jobs:
155155
#install python pip
156156
sudo apt install -y python3 python3-pip pipx
157157
158+
- name: Test CAP_BPF privilege check
159+
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
160+
if: ${{ !contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel) }}
161+
with:
162+
provision: 'false'
163+
cmd: |
164+
set -euxo pipefail
165+
uname -a
166+
cat /etc/issue
167+
pushd /host
168+
bash /host/testdata/run_cap_bpf_test.sh "" "CAP_BPF"
169+
popd
170+
171+
172+
- name: Test CAP_SYS_ADMIN privilege check
173+
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
174+
if: contains(fromJSON('["4.19-20240912.022020", "5.4-20240912.022020"]'), matrix.kernel)
175+
with:
176+
provision: 'false'
177+
cmd: |
178+
set -euxo pipefail
179+
uname -a
180+
cat /etc/issue
181+
pushd /host
182+
bash /host/testdata/run_cap_bpf_test.sh "" "CAP_SYS_ADMIN"
183+
popd
184+
158185
- name: Test filter by comm
159186
uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
160187
with:
@@ -165,9 +192,9 @@ jobs:
165192
cat /etc/issue
166193
pushd /host
167194
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
168-
bash /host/testdata/test_filter_by_comm.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
195+
bash /host/testdata/test_filter_by_comm.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
169196
else
170-
bash /host/testdata/test_filter_by_comm.sh '/host/kyanos/kyanos $kyanos_log_option'
197+
bash /host/testdata/test_filter_by_comm.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
171198
fi
172199
popd
173200
@@ -181,9 +208,9 @@ jobs:
181208
cat /etc/issue
182209
pushd /host
183210
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
184-
bash /host/testdata/test_gotls.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
211+
bash /host/testdata/test_gotls.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
185212
else
186-
bash /host/testdata/test_gotls.sh '/host/kyanos/kyanos $kyanos_log_option'
213+
bash /host/testdata/test_gotls.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
187214
fi
188215
popd
189216
@@ -197,9 +224,9 @@ jobs:
197224
cat /etc/issue
198225
pushd /host
199226
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
200-
bash /host/testdata/test_https.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
227+
bash /host/testdata/test_https.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
201228
else
202-
bash /host/testdata/test_https.sh '/host/kyanos/kyanos $kyanos_log_option'
229+
bash /host/testdata/test_https.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
203230
fi
204231
popd
205232
@@ -213,9 +240,9 @@ jobs:
213240
cat /etc/issue
214241
pushd /host
215242
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
216-
bash /host/testdata/test_side.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
243+
bash /host/testdata/test_side.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
217244
else
218-
bash /host/testdata/test_side.sh '/host/kyanos/kyanos $kyanos_log_option'
245+
bash /host/testdata/test_side.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
219246
fi
220247
popd
221248
@@ -229,9 +256,9 @@ jobs:
229256
cat /etc/issue
230257
pushd /host
231258
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
232-
bash /host/testdata/test_mysql.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
259+
bash /host/testdata/test_mysql.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
233260
else
234-
bash /host/testdata/test_mysql.sh '/host/kyanos/kyanos $kyanos_log_option'
261+
bash /host/testdata/test_mysql.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
235262
fi
236263
popd
237264
@@ -244,9 +271,9 @@ jobs:
244271
uname -a
245272
cat /etc/issue
246273
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
247-
bash /host/testdata/test_base.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
274+
bash /host/testdata/test_base.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
248275
else
249-
bash /host/testdata/test_base.sh '/host/kyanos/kyanos $kyanos_log_option'
276+
bash /host/testdata/test_base.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
250277
fi
251278
252279
@@ -259,9 +286,9 @@ jobs:
259286
uname -a
260287
cat /etc/issue
261288
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
262-
bash /host/testdata/test_filter_by_l4.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
289+
bash /host/testdata/test_filter_by_l4.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
263290
else
264-
bash /host/testdata/test_filter_by_l4.sh '/host/kyanos/kyanos $kyanos_log_option'
291+
bash /host/testdata/test_filter_by_l4.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
265292
fi
266293
267294
- name: Test kern evt
@@ -273,9 +300,9 @@ jobs:
273300
uname -a
274301
cat /etc/issue
275302
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
276-
bash /host/testdata/test_kern_evt.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
303+
bash /host/testdata/test_kern_evt.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
277304
else
278-
bash /host/testdata/test_kern_evt.sh '/host/kyanos/kyanos $kyanos_log_option'
305+
bash /host/testdata/test_kern_evt.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
279306
fi
280307
281308
- name: Test test docker filter by container id
@@ -287,9 +314,9 @@ jobs:
287314
uname -a
288315
cat /etc/issue
289316
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
290-
bash /host/testdata/test_docker_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
317+
bash /host/testdata/test_docker_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
291318
else
292-
bash /host/testdata/test_docker_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option'
319+
bash /host/testdata/test_docker_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
293320
fi
294321
295322
- name: Test test docker filter by container name
@@ -301,9 +328,9 @@ jobs:
301328
uname -a
302329
cat /etc/issue
303330
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
304-
bash /host/testdata/test_docker_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
331+
bash /host/testdata/test_docker_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
305332
else
306-
bash /host/testdata/test_docker_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option'
333+
bash /host/testdata/test_docker_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
307334
fi
308335
309336
- name: Test filter by pid
@@ -315,9 +342,9 @@ jobs:
315342
uname -a
316343
cat /etc/issue
317344
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
318-
bash /host/testdata/test_docker_filter_by_pid.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
345+
bash /host/testdata/test_docker_filter_by_pid.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
319346
else
320-
bash /host/testdata/test_docker_filter_by_pid.sh '/host/kyanos/kyanos $kyanos_log_option'
347+
bash /host/testdata/test_docker_filter_by_pid.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
321348
fi
322349
323350
- name: Test test containerd filter by container name
@@ -329,9 +356,9 @@ jobs:
329356
uname -a
330357
cat /etc/issue
331358
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
332-
bash /host/testdata/test_containerd_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
359+
bash /host/testdata/test_containerd_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
333360
else
334-
bash /host/testdata/test_containerd_filter_by_container_name.sh '/host/kyanos/kyanos $kyanos_log_option'
361+
bash /host/testdata/test_containerd_filter_by_container_name.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
335362
fi
336363
337364
- name: Test test containerd filter by container id
@@ -343,9 +370,9 @@ jobs:
343370
uname -a
344371
cat /etc/issue
345372
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
346-
bash /host/testdata/test_containerd_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
373+
bash /host/testdata/test_containerd_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
347374
else
348-
bash /host/testdata/test_containerd_filter_by_container_id.sh '/host/kyanos/kyanos $kyanos_log_option'
375+
bash /host/testdata/test_containerd_filter_by_container_id.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
349376
fi
350377
351378
- name: Test redis
@@ -357,9 +384,9 @@ jobs:
357384
uname -a
358385
cat /etc/issue
359386
if [ -f "/var/lib/kyanos/btf/current.btf" ]; then
360-
bash /host/testdata/test_redis.sh '/host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
387+
bash /host/testdata/test_redis.sh 'sudo /host/kyanos/kyanos $kyanos_log_option --btf /var/lib/kyanos/btf/current.btf'
361388
else
362-
bash /host/testdata/test_redis.sh '/host/kyanos/kyanos $kyanos_log_option'
389+
bash /host/testdata/test_redis.sh 'sudo /host/kyanos/kyanos $kyanos_log_option'
363390
fi
364391
365392
- name: Test k8s

‎agent/agent.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@ func SetupAgent(options ac.AgentOptions) {
3535
return
3636
}
3737

38-
if os.Geteuid() != 0 {
39-
common.AgentLog.Error("Kyanos requires root privileges to run. Please run kyanos with sudo.")
38+
if ok, err := ac.HasPermission(); err != nil {
39+
common.AgentLog.Error("check capabilities failed: ", err)
40+
return
41+
} else if !ok {
42+
common.AgentLog.Error("Kyanos requires CAP_BPF to run. Please run kyanos with sudo or run container in privilege mode.")
4043
return
4144
}
4245

‎agent/common/permission.go

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package common
2+
3+
import (
4+
"golang.org/x/sys/unix"
5+
)
6+
7+
const (
8+
// capBpf 0000 0000 0000 0000 0000 0000 1000 0000
9+
capBpf = 1 << (unix.CAP_BPF - 32)
10+
// capSysAdmin 0000 0000 0010 0000 0000 0000 0000 0000
11+
capSysAdmin = 1 << unix.CAP_SYS_ADMIN
12+
)
13+
14+
// HasPermission reference: https://man7.org/linux/man-pages/man2/capset.2.html
15+
func HasPermission() (bool, error) {
16+
hdr := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3}
17+
var data [2]unix.CapUserData
18+
if err := unix.Capget(&hdr, &data[0]); err != nil {
19+
return false, err
20+
}
21+
// Note that the CAP_* values are bit indexes and need to be bit-shifted before ORing into the bit fields.
22+
// Note that 64-bit capabilities use datap[0] and datap[1], whereas 32-bit capabilities use only datap[0].
23+
return data[1].Permitted&capBpf != 0 || data[0].Permitted&capSysAdmin != 0, nil
24+
}

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ require (
3434
k8s.io/cri-api v0.31.0
3535
k8s.io/klog/v2 v2.130.1
3636
k8s.io/kubernetes v1.24.17
37+
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
3738
)
3839

3940
require (
@@ -138,7 +139,6 @@ require (
138139
k8s.io/apimachinery v0.31.1 // indirect
139140
k8s.io/apiserver v0.31.1 // indirect
140141
k8s.io/component-base v0.31.1 // indirect
141-
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
142142
rsc.io/binaryregexp v0.2.0 // indirect
143143
)
144144

‎go.sum

-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
3131
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
3232
github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE=
3333
github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU=
34-
github.com/charmbracelet/bubbletea v1.2.2 h1:EMz//Ky/aFS2uLcKqpCst5UOE6z5CFDGRsUpyXz0chs=
35-
github.com/charmbracelet/bubbletea v1.2.2/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM=
3634
github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv3KnQRNCE=
3735
github.com/charmbracelet/bubbletea v1.2.4/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM=
3836
github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=

‎testdata/run_cap_bpf_test.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
DOCKER_REGISTRY="$1"
5+
if [ -n "$DOCKER_REGISTRY" ]; then
6+
# 检查是否以 / 结尾
7+
if [[ "$DOCKER_REGISTRY" != */ ]]; then
8+
DOCKER_REGISTRY="${DOCKER_REGISTRY}/"
9+
fi
10+
else
11+
echo "DOCKER_REGISTRY is missing."
12+
fi
13+
14+
# CAP_SYS_RESOURCE reference: https://docs.ebpf.io/linux/concepts/resource-limit/
15+
sudo docker run -d --ulimit memlock=100000000000:100000000000 --cap-add=CAP_SYS_RESOURCE --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true
16+
sudo docker cp /host/kyanos/kyanos alpine:/
17+
sudo docker cp ./testdata/test_not_add_cap_bpf.sh alpine:/
18+
sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/
19+
sudo docker exec alpine sh -c 'sh /test_not_add_cap_bpf.sh "/kyanos --btf /current.btf"'
20+
sudo docker stop alpine && sudo docker rm alpine
21+
22+
# mount sys reference: https://stackoverflow.com/questions/75808955/error-mounting-sys-kernel-debug-tracing-to-rootfs
23+
sudo docker run -d -v /sys/:/sys/ --privileged --name alpine $DOCKER_REGISTRY'alpine' sh -c 'sleep 120' || true
24+
sudo docker cp /host/kyanos/kyanos alpine:/
25+
sudo docker cp ./testdata/test_add_cap_bpf.sh alpine:/
26+
sudo docker cp /var/lib/kyanos/btf/current.btf alpine:/
27+
sudo docker exec alpine sh -c 'sh /test_add_cap_bpf.sh "/kyanos --btf /current.btf"'
28+
sudo docker stop alpine && sudo docker rm alpine

‎testdata/test_add_cap_bpf.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env sh
2+
set -x
3+
4+
CMD="$1"
5+
FILE_PREFIX="/tmp/kyanos"
6+
LNAME="${FILE_PREFIX}_test_add_cap_bpf_before.log"
7+
8+
timeout 30 ${CMD} watch http --debug-output 2>&1 | tee "${LNAME}" &
9+
wait
10+
11+
cat "${LNAME}"
12+
cat "${LNAME}" | grep -v "requires CAP_BPF"

‎testdata/test_not_add_cap_bpf.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env sh
2+
set -x
3+
4+
CMD="$1"
5+
FILE_PREFIX="/tmp/kyanos"
6+
LNAME="${FILE_PREFIX}_test_not_add_cap_bpf_before.log"
7+
8+
timeout 30 ${CMD} watch http --debug-output 2>&1 | tee "${LNAME}" &
9+
wait
10+
11+
cat "${LNAME}"
12+
cat "${LNAME}" | grep "requires CAP_BPF"

0 commit comments

Comments
 (0)