Skip to content

Commit 39d12ab

Browse files
edwintorokedwintorok
committed
nginx/perf
1 parent d27a118 commit 39d12ab

File tree

4 files changed

+446
-0
lines changed

4 files changed

+446
-0
lines changed

‎scripts/stunnel-perf/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tmp/

‎scripts/stunnel-perf/benchmark.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/bash
2+
# Set a machine into benchmarking mode.
3+
#
4+
# For native Linux, and Intel-only for now (AMD has a different way of disabling turbo)
5+
#
6+
# References: https://llvm.org/docs/Benchmarking.html
7+
8+
# This is for Fedora 42, adapt for your own OS
9+
declare -a packages=(
10+
"cpupower" "tuned" "ss" "turbostat" "lscpu" "lspci" "ip" "ethtool" "jq" "nginx" "hitch" "stunnel" "iperf3" "nuttcp" "httperf" "git" "hyperfine" "varnish" "socat" "curl"
11+
)
12+
13+
# httperf --server $OTHER_IP --port 8080 --num-calls 3000 --num-conns=1 --uri /hello
14+
15+
install_packages()
16+
{
17+
sudo dnf install "${packages[@]}"
18+
}
19+
20+
stop_services()
21+
{
22+
sudo cp /lib/systemd/system/rescue.target /lib/systemd/system/benchmark.target
23+
sudo systemctl add-wants benchmark.target sshd.service
24+
sudo systemctl isolate benchmark.target
25+
sudo systemctl list-units --state=running
26+
}
27+
28+
start_services()
29+
{
30+
sudo systemctl isolate multi-user.target
31+
}
32+
33+
set_cpu()
34+
{
35+
# Prevent frequency-scaling
36+
sudo cpupower frequency-set -g performance
37+
38+
# Prevent entering deep C-states to reduce latency and avoid menu governor bugs (there was one recently)
39+
# Although if we completely disable C1 then we are going to hit RAPL power limits immediately,
40+
# which results in a 30% perf hit on single-stream iperf
41+
# Disable just deeper C states, which avoids the big perf hit (although now we're exposed to menu governor bugs)
42+
sudo cpupower idle-set -D 2
43+
44+
# Disable Turbo Boost for more stable measurements
45+
# cpupower set --turbo-boost 0 doesn't work when the driver is intel_pstate instead of cpufreq
46+
sudo sh -c 'echo 1 >/sys/devices/system/cpu/intel_pstate/no_turbo'
47+
48+
# AMD
49+
sudo cpupower set -m passive
50+
sudo cpupower set --turbo-boost 0
51+
52+
sudo cpupower -c all set --perf-bias 0
53+
54+
# Bring system into a consistent state
55+
sudo tuned-adm profile network-throughput
56+
57+
# we could use tuned-adm verify, but there is a bug where it tries to read non-existent hung_task_timeout_secs
58+
# that only fails on network-latency, not network-throughput though
59+
sudo tuned-adm verify
60+
}
61+
62+
describe()
63+
{
64+
# Check all the settings we've changed above
65+
66+
# List running processes
67+
set -x
68+
sudo systemctl list-units --state=running
69+
LIBPROC_HIDE_KERNEL=1 ps -ejH
70+
71+
sudo cpupower -c all frequency-info
72+
sudo cpupower -c all idle-info
73+
sudo cpupower -c all info
74+
sudo cat /sys/devices/system/cpu/intel_pstate/no_turbo
75+
sudo tuned-adm profile_info
76+
sudo ss -neopatium
77+
78+
lscpu
79+
sudo turbostat true
80+
sudo lspci -k
81+
sudo ip a
82+
83+
for P in /sys/class/net/*; do
84+
IFACE=$(basename "${P}")
85+
sudo ethtool "${IFACE}"
86+
sudo ethtool -k "${IFACE}"
87+
done
88+
89+
sudo sos report --batch
90+
91+
set +x
92+
}
93+
94+
open_ports()
95+
{
96+
# iperf3, nuttcp
97+
for P in 5201 5000 5101 9080 9443 9444 9445 9446; do
98+
sudo firewall-cmd --add-port "${P}/tcp"
99+
done
100+
}
101+
102+
iperf3_server()
103+
{
104+
iperf3 -s
105+
}
106+
107+
iperf3_client()
108+
{
109+
# TODO: autodetect IP
110+
iperf3 --json --logfile iperf3.json -c 10.70.58.43
111+
iperf3 --json --logfile iperf3_2.json -c 10.70.58.43 -P2
112+
iperf3 --json --logfile iperf3_4.json -c 10.70.58.43 -P4
113+
}

‎scripts/stunnel-perf/l

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
+ set -eu
2+
+ TLS_CIPHERS=ECDHE-RSA-AES256-GCM-SHA384
3+
+ TLS_PROTO=TLSv1.2
4+
+ NGINX_PORT=9080
5+
+ NGINX_TLS_PORT=9443
6+
+ STUNNEL_TLS_PORT=9444
7+
+ HITCH_TLS_PORT=9445
8+
+ SOCAT_TLS_PORT=9446
9+
+ REPEAT=1
10+
+ TEST_SIZE=1MiB
11+
+ PEM=stunnel.pem
12+
+ TESTDIR=/home/edvint/git/xen-api/scripts/stunnel-perf/tmp
13+
+ RAMDIR=/dev/shm/stunnel-test
14+
+ mkdir -p /dev/shm/stunnel-test /home/edvint/git/xen-api/scripts/stunnel-perf/tmp
15+
+ cd /home/edvint/git/xen-api/scripts/stunnel-perf/tmp
16+
+ test -f stunnel.pem
17+
+ cat
18+
+ truncate --size 1MiB /dev/shm/stunnel-test/1MiB
19+
+ cat
20+
+ cat
21+
+ killall -v -9 stunnel nginx hitch socat
22+
Killed nginx(944325) with signal 9
23+
Killed nginx(944326) with signal 9
24+
Killed socat(944327) with signal 9
25+
Killed stunnel(944330) with signal 9
26+
Killed hitch(944334) with signal 9
27+
Killed hitch(944335) with signal 9
28+
Killed hitch(944337) with signal 9
29+
Killed stunnel(944340) with signal 9
30+
Killed socat(944343) with signal 9
31+
Killed stunnel(944347) with signal 9
32+
Killed socat(944350) with signal 9
33+
Killed stunnel(944354) with signal 9
34+
Killed socat(944356) with signal 9
35+
Killed stunnel(944361) with signal 9
36+
Killed socat(944363) with signal 9
37+
+ nginx -c /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/nginx.conf
38+
nginx: [alert] could not open error log file: open() "/var/log/nginx/error.log" failed (13: Permission denied)
39+
+ stunnel /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/stunnel.conf
40+
+ socat -b65536 OPENSSL-LISTEN:9446,fork,reuseaddr,cert=stunnel.pem,verify=0,cipher=ECDHE-RSA-AES256-GCM-SHA384,compress=none,reuseport TCP4:127.0.0.1:9080
41+
+ hitch --config /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/hitch.conf
42+
20250603T180955.914523 [944519] {core} hitch 1.7.2 starting
43+
20250603T180955.914555 [944519] WARNING: {core} OpenSSL version mismatch; hitch was compiled with 30000010, now using 30200020.
44+
20250603T180955.917361 [944519] {core} Loading certificate pem files (1)
45+
+ DEST=127.0.0.1
46+
+ for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}
47+
+ PORT2=19443
48+
+ cat
49+
+ stunnel /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/stunnel-client-9443.conf
50+
+ PORT3=29443
51+
+ for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}
52+
+ PORT2=19444
53+
+ cat
54+
+ socat -b65536 TCP4-LISTEN:29443,fork,bind=127.0.0.1,reuseport OPENSSL:127.0.0.1:9443,cipher=ECDHE-RSA-AES256-GCM-SHA384,verify=0
55+
+ stunnel /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/stunnel-client-9444.conf
56+
+ PORT3=29444
57+
+ for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}
58+
+ PORT2=19445
59+
+ cat
60+
+ socat -b65536 TCP4-LISTEN:29444,fork,bind=127.0.0.1,reuseport OPENSSL:127.0.0.1:9444,cipher=ECDHE-RSA-AES256-GCM-SHA384,verify=0
61+
+ stunnel /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/stunnel-client-9445.conf
62+
+ PORT3=29445
63+
+ for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}
64+
+ PORT2=19446
65+
+ cat
66+
+ socat -b65536 TCP4-LISTEN:29445,fork,bind=127.0.0.1,reuseport OPENSSL:127.0.0.1:9445,cipher=ECDHE-RSA-AES256-GCM-SHA384,verify=0
67+
+ stunnel /home/edvint/git/xen-api/scripts/stunnel-perf/tmp/stunnel-client-9446.conf
68+
+ PORT3=29446
69+
+ measure nginx http://127.0.0.1:9080 '' plain
70+
+ echo
71+
+ echo 'plain: curl --> nginx http://127.0.0.1:9080 '
72+
+ socat -b65536 TCP4-LISTEN:29446,fork,bind=127.0.0.1,reuseport OPENSSL:127.0.0.1:9446,cipher=ECDHE-RSA-AES256-GCM-SHA384,verify=0
73+
++ seq 1
74+
+ for i in $(seq "${REPEAT}")
75+
+ curl -k -o /dev/null http://127.0.0.1:9080/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
76+
+ tee -a plain.csv
77+
% Total % Received % Xferd Average Speed Time Time Time Current
78+
Dload Upload Total Spent Left Speed
79+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 1000M 0 --:--:-- --:--:-- --:--:-- 1000M
80+
+ python3
81+
+ NGINX_URL=http://127.0.0.1:9080
82+
+ measure nginx-ssl https://127.0.0.1:9443 '' nginx-ssl
83+
+ echo
84+
+ echo 'nginx-ssl: curl --> nginx-ssl https://127.0.0.1:9443 '
85+
++ seq 1
86+
+ for i in $(seq "${REPEAT}")
87+
+ curl -k -o /dev/null https://127.0.0.1:9443/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
88+
+ tee -a nginx-ssl.csv
89+
% Total % Received % Xferd Average Speed Time Time Time Current
90+
Dload Upload Total Spent Left Speed
91+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 45.4M 0 --:--:-- --:--:-- --:--:-- 47.6M
92+
+ python3
93+
+ measure stunnel https://127.0.0.1:9444 '-> nginx http://127.0.0.1:9080' stunnel_server
94+
+ echo
95+
+ echo 'stunnel_server: curl --> stunnel https://127.0.0.1:9444 -> nginx http://127.0.0.1:9080'
96+
++ seq 1
97+
+ for i in $(seq "${REPEAT}")
98+
+ curl -k -o /dev/null https://127.0.0.1:9444/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
99+
+ tee -a stunnel_server.csv
100+
% Total % Received % Xferd Average Speed Time Time Time Current
101+
Dload Upload Total Spent Left Speed
102+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 71.4M 0 --:--:-- --:--:-- --:--:-- 71.4M
103+
+ python3
104+
+ measure hitch https://127.0.0.1:9445 '-> nginx http://127.0.0.1:9080' hitch_server
105+
+ echo
106+
+ echo 'hitch_server: curl --> hitch https://127.0.0.1:9445 -> nginx http://127.0.0.1:9080'
107+
++ seq 1
108+
+ for i in $(seq "${REPEAT}")
109+
+ tee -a hitch_server.csv
110+
+ curl -k -o /dev/null https://127.0.0.1:9445/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
111+
% Total % Received % Xferd Average Speed Time Time Time Current
112+
Dload Upload Total Spent Left Speed
113+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 71.4M 0 --:--:-- --:--:-- --:--:-- 71.4M
114+
+ python3
115+
+ measure socat https://127.0.0.1:9446 '-> nginx http://127.0.0.1:9080' socat_server
116+
+ echo
117+
+ echo 'socat_server: curl --> socat https://127.0.0.1:9446 -> nginx http://127.0.0.1:9080'
118+
++ seq 1
119+
+ for i in $(seq "${REPEAT}")
120+
+ curl -k -o /dev/null https://127.0.0.1:9446/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
121+
+ tee -a socat_server.csv
122+
% Total % Received % Xferd Average Speed Time Time Time Current
123+
Dload Upload Total Spent Left Speed
124+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 17.8M 0 --:--:-- --:--:-- --:--:-- 18.1M
125+
+ python3
126+
+ measure 'stunnel(client)' http://127.0.0.1:19444 '-> stunnel https://127.0.0.1:9444 -> nginx http://127.0.0.1:9080' stunnel_client_nginx
127+
+ echo
128+
+ echo 'stunnel_client_nginx: curl --> stunnel(client) http://127.0.0.1:19444 -> stunnel https://127.0.0.1:9444 -> nginx http://127.0.0.1:9080'
129+
++ seq 1
130+
+ for i in $(seq "${REPEAT}")
131+
+ curl -k -o /dev/null http://127.0.0.1:19444/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
132+
+ tee -a stunnel_client_nginx.csv
133+
% Total % Received % Xferd Average Speed Time Time Time Current
134+
Dload Upload Total Spent Left Speed
135+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 166M 0 --:--:-- --:--:-- --:--:-- 166M
136+
+ python3
137+
+ measure 'stunnel(client)' http://127.0.0.1:19445 '-> hitch https://127.0.0.1:9445 -> nginx http://127.0.0.1:9080' stunnel_client_hitch
138+
+ echo
139+
+ echo 'stunnel_client_hitch: curl --> stunnel(client) http://127.0.0.1:19445 -> hitch https://127.0.0.1:9445 -> nginx http://127.0.0.1:9080'
140+
++ seq 1
141+
+ for i in $(seq "${REPEAT}")
142+
+ tee -a stunnel_client_hitch.csv
143+
+ curl -k -o /dev/null http://127.0.0.1:19445/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
144+
% Total % Received % Xferd Average Speed Time Time Time Current
145+
Dload Upload Total Spent Left Speed
146+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 166M 0 --:--:-- --:--:-- --:--:-- 166M
147+
+ python3
148+
+ measure 'stunnel(client)' http://127.0.0.1:19446 '-> socat https://127.0.0.1:9446 -> nginx http://127.0.0.1:9080' stunnel_client_socat
149+
+ echo
150+
+ echo 'stunnel_client_socat: curl --> stunnel(client) http://127.0.0.1:19446 -> socat https://127.0.0.1:9446 -> nginx http://127.0.0.1:9080'
151+
++ seq 1
152+
+ for i in $(seq "${REPEAT}")
153+
+ curl -k -o /dev/null http://127.0.0.1:19446/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
154+
+ tee -a stunnel_client_socat.csv
155+
% Total % Received % Xferd Average Speed Time Time Time Current
156+
Dload Upload Total Spent Left Speed
157+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 125M 0 --:--:-- --:--:-- --:--:-- 125M
158+
+ python3
159+
+ measure 'socat(client)' http://127.0.0.1:29444 '-> stunnel https://127.0.0.1:9444 -> nginx http://127.0.0.1:9080' socat_client_stunnel
160+
+ echo
161+
+ echo 'socat_client_stunnel: curl --> socat(client) http://127.0.0.1:29444 -> stunnel https://127.0.0.1:9444 -> nginx http://127.0.0.1:9080'
162+
++ seq 1
163+
+ for i in $(seq "${REPEAT}")
164+
+ curl -k -o /dev/null http://127.0.0.1:29444/1MiB --write-out '%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n'
165+
+ tee -a socat_client_stunnel.csv
166+
% Total % Received % Xferd Average Speed Time Time Time Current
167+
Dload Upload Total Spent Left Speed
168+
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0100 1024k 100 1024k 0 0 55.5M 0 --:--:-- --:--:-- --:--:-- 55.5M
169+
+ python3
170+
2025/06/03 18:10:16 socat[944798] E write(6, 0x7f6e16887000, 16384): Broken pipe

0 commit comments

Comments
 (0)