Skip to content

Commit c71f846

Browse files
edwintorokedwintorok
committed
nginx/perf
1 parent d27a118 commit c71f846

File tree

3 files changed

+257
-0
lines changed

3 files changed

+257
-0
lines changed

‎scripts/stunnel-perf/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tmp/

‎scripts/stunnel-perf/benchmark.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/bash
2+
# Set a machine into benchmarking mode.
3+
#
4+
# For native Linux, and Intel-only for now (AMD has a different way of disabling turbo)
5+
#
6+
# References: https://llvm.org/docs/Benchmarking.html
7+
8+
# This is for Fedora 42, adapt for your own OS
9+
declare -a packages=(
10+
"cpupower" "tuned" "ss" "turbostat" "lscpu" "lspci" "ip" "ethtool" "jq" "nginx" "hitch" "stunnel" "iperf3" "nuttcp" "httperf" "git" "hyperfine" "varnish" "socat" "curl"
11+
)
12+
13+
# httperf --server $OTHER_IP --port 8080 --num-calls 3000 --num-conns=1 --uri /hello
14+
15+
install_packages()
16+
{
17+
sudo dnf install "${packages[@]}"
18+
}
19+
20+
stop_services()
21+
{
22+
sudo cp /lib/systemd/system/rescue.target /lib/systemd/system/benchmark.target
23+
sudo systemctl add-wants benchmark.target sshd.service
24+
sudo systemctl isolate benchmark.target
25+
sudo systemctl list-units --state=running
26+
}
27+
28+
start_services()
29+
{
30+
sudo systemctl isolate multi-user.target
31+
}
32+
33+
set_cpu()
34+
{
35+
# Prevent frequency-scaling
36+
sudo cpupower frequency-set -g performance
37+
38+
# Prevent entering deep C-states to reduce latency and avoid menu governor bugs (there was one recently)
39+
# Although if we completely disable C1 then we are going to hit RAPL power limits immediately,
40+
# which results in a 30% perf hit on single-stream iperf
41+
# Disable just deeper C states, which avoids the big perf hit (although now we're exposed to menu governor bugs)
42+
sudo cpupower idle-set -D 2
43+
44+
# Disable Turbo Boost for more stable measurements
45+
# cpupower set --turbo-boost 0 doesn't work when the driver is intel_pstate instead of cpufreq
46+
sudo sh -c 'echo 1 >/sys/devices/system/cpu/intel_pstate/no_turbo'
47+
48+
# AMD
49+
sudo cpupower set -m passive
50+
sudo cpupower set --turbo-boost 0
51+
52+
sudo cpupower -c all set --perf-bias 0
53+
54+
# Bring system into a consistent state
55+
sudo tuned-adm profile network-throughput
56+
57+
# we could use tuned-adm verify, but there is a bug where it tries to read non-existent hung_task_timeout_secs
58+
# that only fails on network-latency, not network-throughput though
59+
sudo tuned-adm verify
60+
}
61+
62+
describe()
63+
{
64+
# Check all the settings we've changed above
65+
66+
# List running processes
67+
set -x
68+
sudo systemctl list-units --state=running
69+
LIBPROC_HIDE_KERNEL=1 ps -ejH
70+
71+
sudo cpupower -c all frequency-info
72+
sudo cpupower -c all idle-info
73+
sudo cpupower -c all info
74+
sudo cat /sys/devices/system/cpu/intel_pstate/no_turbo
75+
sudo tuned-adm profile_info
76+
sudo ss -neopatium
77+
78+
lscpu
79+
sudo turbostat true
80+
sudo lspci -k
81+
sudo ip a
82+
83+
for P in /sys/class/net/*; do
84+
IFACE=$(basename "${P}")
85+
sudo ethtool "${IFACE}"
86+
sudo ethtool -k "${IFACE}"
87+
done
88+
89+
sudo sos report --batch
90+
91+
set +x
92+
}
93+
94+
open_ports()
95+
{
96+
# iperf3, nuttcp
97+
for P in 5201 5000 5101 9080 9443 9444 9445 9446; do
98+
sudo firewall-cmd --add-port "${P}/tcp"
99+
done
100+
}
101+
102+
iperf3_server()
103+
{
104+
iperf3 -s
105+
}
106+
107+
iperf3_client()
108+
{
109+
# TODO: autodetect IP
110+
iperf3 --json --logfile iperf3.json -c 10.70.58.43
111+
iperf3 --json --logfile iperf3_2.json -c 10.70.58.43 -P2
112+
iperf3 --json --logfile iperf3_4.json -c 10.70.58.43 -P4
113+
}

‎scripts/stunnel-perf/server.sh

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
#!/bin/sh
2+
set -eu
3+
4+
# Use a single, fixed cipher for consistent tests across different tools
5+
TLS_CIPHERS=ECDHE-RSA-AES256-GCM-SHA384
6+
TLS_PROTO=TLSv1.2
7+
8+
# Use non-root ports
9+
NGINX_PORT=9080
10+
NGINX_TLS_PORT=9443
11+
STUNNEL_TLS_PORT=9444
12+
HITCH_TLS_PORT=9445
13+
SOCAT_TLS_PORT=9446
14+
15+
REPEAT=${REPEAT-30}
16+
17+
PEM=stunnel.pem
18+
TESTDIR="${PWD}/tmp"
19+
RAMDIR=/dev/shm/stunnel-test
20+
mkdir -p "${RAMDIR}" "${TESTDIR}"
21+
22+
cd "${TESTDIR}"
23+
24+
test -f "${PEM}" || openssl req -new -subj "/C=GB/CN=foo.example.com" -days 10 -x509 -noenc -batch -out "${PEM}" -keyout "${PEM}" -noenc -batch
25+
26+
cat >stunnel.conf <<EOF
27+
fips=no
28+
socket = r:TCP_NODELAY=1
29+
socket = a:TCP_NODELAY=1
30+
socket = l:TCP_NODELAY=1
31+
socket = r:SO_KEEPALIVE=1
32+
socket = a:SO_KEEPALIVE=1
33+
34+
[test]
35+
accept = :::${STUNNEL_TLS_PORT}
36+
cert = ${PEM}
37+
connect = ${NGINX_PORT}
38+
ciphers = ${TLS_CIPHERS}
39+
options = CIPHER_SERVER_PREFERENCE
40+
sslVersion = ${TLS_PROTO}
41+
EOF
42+
43+
truncate --size 1GiB "${RAMDIR}/1GiB"
44+
45+
cat >nginx.conf <<EOF
46+
error_log stderr error;
47+
pid ${TESTDIR}/nginx.pid;
48+
daemon on;
49+
events { }
50+
http {
51+
client_body_temp_path "${TESTDIR}/nginx-client_body";
52+
proxy_temp_path "${TESTDIR}/nginx-proxy";
53+
fastcgi_temp_path "${TESTDIR}/nginx-fcgi";
54+
uwsgi_temp_path ${TESTDIR}/nginx-uwsgi";
55+
scgi_temp_path ${TESTDIR}/nginx-scgi";
56+
access_log off;
57+
tcp_nodelay on;
58+
sendfile on;
59+
server {
60+
gzip off;
61+
listen "${NGINX_PORT}" reuseport;
62+
listen "${NGINX_TLS_PORT}" reuseport ssl;
63+
ssl_certificate "${PEM}";
64+
ssl_certificate_key "${PEM}";
65+
ssl_protocols "${TLS_PROTO}";
66+
ssl_ciphers "${TLS_CIPHERS}";
67+
root "${RAMDIR}";
68+
location = /hello {
69+
default_type text/plain;
70+
return 200 "Hello";
71+
}
72+
}
73+
}
74+
EOF
75+
76+
cat >hitch.conf <<EOF
77+
backend="[127.0.0.1]:${NGINX_PORT}"
78+
ciphers="${TLS_CIPHERS}"
79+
daemon=on
80+
frontend={
81+
host="*"
82+
port="${HITCH_TLS_PORT}"
83+
pem-file="${PEM}"
84+
prefer-server-ciphers=on
85+
tls-protos=${TLS_PROTO}
86+
}
87+
EOF
88+
89+
killall -v -9 stunnel nginx hitch socat || :
90+
91+
# Start servers
92+
nginx -c "${TESTDIR}/nginx.conf"
93+
socat -b65536 "OPENSSL-LISTEN:${SOCAT_TLS_PORT},fork,reuseaddr,cert=${PEM},verify=0,cipher=${TLS_CIPHERS},compress=none,reuseport" "TCP4:127.0.0.1:${NGINX_PORT}" &
94+
stunnel "${TESTDIR}/stunnel.conf"
95+
hitch --config "${TESTDIR}/hitch.conf"
96+
97+
DEST=${DEST:-127.0.0.1}
98+
99+
# Start clients (they could also use a pipe, for simplicity use a local port)
100+
for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}; do
101+
PORT2=$(( "${PORT}" + 10000 ))
102+
cat >"stunnel-client-${PORT}.conf" <<EOF
103+
client=yes
104+
foreground=no
105+
socket=r:TCP_NODELAY=1
106+
socket=r:SO_KEEPALIVE=1
107+
socket=a:SO_KEEPALIVE=1
108+
fips=no
109+
connect=${DEST}:${PORT}
110+
sslVersion=${TLS_PROTO}
111+
ciphers=${TLS_CIPHERS}
112+
[client-proxy]
113+
accept=127.0.0.1:${PORT2}
114+
EOF
115+
stunnel "${TESTDIR}/stunnel-client-${PORT}.conf"
116+
PORT3=$(( ${PORT} + 20000 ))
117+
socat -b65536 "TCP4-LISTEN:${PORT3},bind=127.0.0.1,reuseport" "OPENSSL:${DEST}:${PORT},cipher=${TLS_CIPHERS},verify=0"&
118+
done
119+
120+
121+
measure()
122+
{
123+
echo
124+
echo "curl --> ${1} ${2} ${3}"
125+
for i in $(seq "${REPEAT}"); do
126+
curl -k -o /dev/null "${2}/1GiB" --write-out "%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n"
127+
done
128+
echo
129+
}
130+
131+
measure "nginx" "http://${DEST}:${NGINX_PORT}" ""
132+
133+
NGINX_URL="http://127.0.0.1:${NGINX_PORT}"
134+
135+
measure "nginx-ssl" "https://${DEST}:${NGINX_TLS_PORT}" ""
136+
137+
measure "stunnel" "https://${DEST}:${STUNNEL_TLS_PORT}" "-> nginx ${NGINX_URL}"
138+
measure "hitch" "https://${DEST}:${HITCH_TLS_PORT}" "-> nginx ${NGINX_URL}"
139+
measure "socat" "https://${DEST}:${SOCAT_TLS_PORT}" "-> nginx ${NGINX_URL}"
140+
141+
measure "stunnel(client)" "http://127.0.0.1:1${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}"
142+
measure "stunnel(client)" "http://127.0.0.1:1${HITCH_TLS_PORT}" "-> hitch https://${DEST}:${HITCH_TLS_PORT} -> nginx ${NGINX_URL}"
143+
measure "stunnel(client)" "http://127.0.0.1:1${SOCAT_TLS_PORT}" "-> socat https://${DEST}:${SOCAT_TLS_PORT} -> nginx ${NGINX_URL}"

0 commit comments

Comments
 (0)