Skip to content

Commit 0af20ac

Browse files
edwintorokedwintorok
committed
nginx/perf
1 parent d27a118 commit 0af20ac

File tree

3 files changed

+252
-0
lines changed

3 files changed

+252
-0
lines changed

‎scripts/stunnel-perf/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tmp/

‎scripts/stunnel-perf/benchmark.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/bash
2+
# Set a machine into benchmarking mode.
3+
#
4+
# For native Linux, and Intel-only for now (AMD has a different way of disabling turbo)
5+
#
6+
# References: https://llvm.org/docs/Benchmarking.html
7+
8+
# This is for Fedora 42, adapt for your own OS
9+
declare -a packages=(
10+
"cpupower" "tuned" "ss" "turbostat" "lscpu" "lspci" "ip" "ethtool" "jq" "nginx" "hitch" "stunnel" "iperf3" "nuttcp" "httperf" "git" "hyperfine" "varnish" "socat" "curl"
11+
)
12+
13+
# httperf --server $OTHER_IP --port 8080 --num-calls 3000 --num-conns=1 --uri /hello
14+
15+
install_packages()
16+
{
17+
sudo dnf install "${packages[@]}"
18+
}
19+
20+
stop_services()
21+
{
22+
sudo cp /lib/systemd/system/rescue.target /lib/systemd/system/benchmark.target
23+
sudo systemctl add-wants benchmark.target sshd.service
24+
sudo systemctl isolate benchmark.target
25+
sudo systemctl list-units --state=running
26+
}
27+
28+
start_services()
29+
{
30+
sudo systemctl isolate multi-user.target
31+
}
32+
33+
set_cpu()
34+
{
35+
# Prevent frequency-scaling
36+
sudo cpupower frequency-set -g performance
37+
38+
# Prevent entering deep C-states to reduce latency and avoid menu governor bugs (there was one recently)
39+
# Although if we completely disable C1 then we are going to hit RAPL power limits immediately,
40+
# which results in a 30% perf hit on single-stream iperf
41+
# Disable just deeper C states, which avoids the big perf hit (although now we're exposed to menu governor bugs)
42+
sudo cpupower idle-set -D 2
43+
44+
# Disable Turbo Boost for more stable measurements
45+
# cpupower set --turbo-boost 0 doesn't work when the driver is intel_pstate instead of cpufreq
46+
sudo sh -c 'echo 1 >/sys/devices/system/cpu/intel_pstate/no_turbo'
47+
48+
# AMD
49+
sudo cpupower set -m passive
50+
sudo cpupower set --turbo-boost 0
51+
52+
sudo cpupower -c all set --perf-bias 0
53+
54+
# Bring system into a consistent state
55+
sudo tuned-adm profile network-throughput
56+
57+
# we could use tuned-adm verify, but there is a bug where it tries to read non-existent hung_task_timeout_secs
58+
# that only fails on network-latency, not network-throughput though
59+
sudo tuned-adm verify
60+
}
61+
62+
describe()
63+
{
64+
# Check all the settings we've changed above
65+
66+
# List running processes
67+
set -x
68+
sudo systemctl list-units --state=running
69+
LIBPROC_HIDE_KERNEL=1 ps -ejH
70+
71+
sudo cpupower -c all frequency-info
72+
sudo cpupower -c all idle-info
73+
sudo cpupower -c all info
74+
sudo cat /sys/devices/system/cpu/intel_pstate/no_turbo
75+
sudo tuned-adm profile_info
76+
sudo ss -neopatium
77+
78+
lscpu
79+
sudo turbostat true
80+
sudo lspci -k
81+
sudo ip a
82+
83+
for P in /sys/class/net/*; do
84+
IFACE=$(basename "${P}")
85+
sudo ethtool "${IFACE}"
86+
sudo ethtool -k "${IFACE}"
87+
done
88+
89+
sudo sos report --batch
90+
91+
set +x
92+
}
93+
94+
open_ports()
95+
{
96+
# iperf3, nuttcp
97+
for P in 5201 5000 5101 9080 9443 9444 9445 9446; do
98+
sudo firewall-cmd --add-port "${P}/tcp"
99+
done
100+
}
101+
102+
iperf3_server()
103+
{
104+
iperf3 -s
105+
}
106+
107+
iperf3_client()
108+
{
109+
# TODO: autodetect IP
110+
iperf3 --json --logfile iperf3.json -c 10.70.58.43
111+
iperf3 --json --logfile iperf3_2.json -c 10.70.58.43 -P2
112+
iperf3 --json --logfile iperf3_4.json -c 10.70.58.43 -P4
113+
}

‎scripts/stunnel-perf/server.sh

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#!/bin/sh
2+
set -eu
3+
4+
# Use a single, fixed cipher for consistent tests across different tools
5+
TLS_CIPHERS=ECDHE-RSA-AES256-GCM-SHA384
6+
TLS_PROTO=TLSv1.2
7+
8+
# Use non-root ports
9+
NGINX_PORT=9080
10+
NGINX_TLS_PORT=9443
11+
STUNNEL_TLS_PORT=9444
12+
HITCH_TLS_PORT=9445
13+
SOCAT_TLS_PORT=9446
14+
15+
PEM=stunnel.pem
16+
TESTDIR="${PWD}/tmp"
17+
RAMDIR=/dev/shm/stunnel-test
18+
mkdir -p "${RAMDIR}" "${TESTDIR}"
19+
20+
cd "${TESTDIR}"
21+
22+
test -f "${PEM}" || openssl req -new -subj "/C=GB/CN=foo.example.com" -days 10 -x509 -noenc -batch -out "${PEM}" -keyout "${PEM}" -noenc -batch
23+
24+
cat >stunnel.conf <<EOF
25+
fips=no
26+
socket = r:TCP_NODELAY=1
27+
socket = a:TCP_NODELAY=1
28+
socket = l:TCP_NODELAY=1
29+
socket = r:SO_KEEPALIVE=1
30+
socket = a:SO_KEEPALIVE=1
31+
32+
[test]
33+
accept = :::${STUNNEL_TLS_PORT}
34+
cert = ${PEM}
35+
connect = ${NGINX_PORT}
36+
ciphers = ${TLS_CIPHERS}
37+
options = CIPHER_SERVER_PREFERENCE
38+
sslVersion = ${TLS_PROTO}
39+
EOF
40+
41+
truncate --size 1GiB "${RAMDIR}/1GiB"
42+
43+
cat >nginx.conf <<EOF
44+
error_log stderr error;
45+
pid ${TESTDIR}/nginx.pid;
46+
daemon on;
47+
events { }
48+
http {
49+
client_body_temp_path "${TESTDIR}/nginx-client_body";
50+
proxy_temp_path "${TESTDIR}/nginx-proxy";
51+
fastcgi_temp_path "${TESTDIR}/nginx-fcgi";
52+
uwsgi_temp_path ${TESTDIR}/nginx-uwsgi";
53+
scgi_temp_path ${TESTDIR}/nginx-scgi";
54+
access_log off;
55+
tcp_nodelay on;
56+
sendfile on;
57+
server {
58+
gzip off;
59+
listen "${NGINX_PORT}" reuseport;
60+
listen "${NGINX_TLS_PORT}" reuseport ssl;
61+
ssl_certificate "${PEM}";
62+
ssl_certificate_key "${PEM}";
63+
ssl_protocols "${TLS_PROTO}";
64+
ssl_ciphers "${TLS_CIPHERS}";
65+
root "${RAMDIR}";
66+
location = /hello {
67+
default_type text/plain;
68+
return 200 "Hello";
69+
}
70+
}
71+
}
72+
EOF
73+
74+
cat >hitch.conf <<EOF
75+
backend="[127.0.0.1]:${NGINX_PORT}"
76+
ciphers="${TLS_CIPHERS}"
77+
daemon=on
78+
frontend={
79+
host="*"
80+
port="${HITCH_TLS_PORT}"
81+
pem-file="${PEM}"
82+
prefer-server-ciphers=on
83+
tls-protos=${TLS_PROTO}
84+
}
85+
EOF
86+
87+
killall -v -9 stunnel nginx hitch socat || :
88+
89+
# Start servers
90+
nginx -c "${TESTDIR}/nginx.conf"
91+
socat -b65536 "OPENSSL-LISTEN:${SOCAT_TLS_PORT},fork,reuseaddr,cert=${PEM},verify=0,cipher=${TLS_CIPHERS},compress=none,reuseport" "TCP4:127.0.0.1:${NGINX_PORT}" &
92+
stunnel "${TESTDIR}/stunnel.conf"
93+
hitch --config "${TESTDIR}/hitch.conf"
94+
95+
DEST=${DEST:-127.0.0.1}
96+
97+
# Start clients (they could also use a pipe, for simplicity use a local port)
98+
for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}; do
99+
PORT2=$(( "${PORT}" + 10000 ))
100+
cat >"stunnel-client-${PORT}.conf" <<EOF
101+
client=yes
102+
foreground=no
103+
socket=r:TCP_NODELAY=1
104+
socket=r:SO_KEEPALIVE=1
105+
socket=a:SO_KEEPALIVE=1
106+
fips=no
107+
connect=${DEST}:${PORT}
108+
sslVersion=${TLS_PROTO}
109+
ciphers=${TLS_CIPHERS}
110+
[client-proxy]
111+
accept=127.0.0.1:${PORT2}
112+
EOF
113+
stunnel "${TESTDIR}/stunnel-client-${PORT}.conf"
114+
PORT3=$(( ${PORT} + 20000 ))
115+
socat -b65536 "TCP4-LISTEN:${PORT3},bind=127.0.0.1,reuseport" "OPENSSL:${DEST}:${PORT},cipher=${TLS_CIPHERS},verify=0"&
116+
done
117+
118+
119+
measure()
120+
{
121+
echo "curl --> ${1} ${2} ${3}"
122+
# curl -o /dev/null "${2}/1GiB" --write-out "%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n"
123+
124+
}
125+
126+
measure "nginx" "http://${DEST}:${NGINX_PORT}" ""
127+
128+
NGINX_URL="http://127.0.0.1:${NGINX_PORT}"
129+
130+
measure "nginx-ssl" "https://${DEST}:${NGINX_TLS_PORT}" ""
131+
132+
measure "stunnel" "https://${DEST}:${STUNNEL_TLS_PORT}" "-> nginx ${NGINX_URL}"
133+
measure "hitch" "https://${DEST}:${HITCH_TLS_PORT}" "-> nginx ${NGINX_URL}"
134+
measure "socat" "https://${DEST}:${SOCAT_TLS_PORT}" "-> nginx ${NGINX_URL}"
135+
136+
measure "stunnel(client)" "http://127.0.0.1:1${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}"
137+
measure "stunnel(client)" "http://127.0.0.1:1${HITCH_TLS_PORT}" "-> hitch https://${DEST}:${HITCH_TLS_PORT} -> nginx ${NGINX_URL}"
138+
measure "stunnel(client)" "http://127.0.0.1:1${SOCAT_TLS_PORT}" "-> socat https://${DEST}:${SOCAT_TLS_PORT} -> nginx ${NGINX_URL}"

0 commit comments

Comments
 (0)