Skip to content

Commit 5a319ed

Browse files
edwintorokedwintorok
committed
nginx/perf
1 parent d27a118 commit 5a319ed

File tree

3 files changed

+306
-0
lines changed

3 files changed

+306
-0
lines changed

‎scripts/stunnel-perf/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tmp/

‎scripts/stunnel-perf/benchmark.sh

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/bin/bash
2+
# Set a machine into benchmarking mode.
3+
#
4+
# For native Linux, and Intel-only for now (AMD has a different way of disabling turbo)
5+
#
6+
# References: https://llvm.org/docs/Benchmarking.html
7+
8+
# This is for Fedora 42, adapt for your own OS
9+
declare -a packages=(
10+
"cpupower" "tuned" "ss" "turbostat" "lscpu" "lspci" "ip" "ethtool" "jq" "nginx" "hitch" "stunnel" "iperf3" "nuttcp" "httperf" "git" "hyperfine" "varnish" "socat" "curl"
11+
)
12+
13+
# httperf --server $OTHER_IP --port 8080 --num-calls 3000 --num-conns=1 --uri /hello
14+
15+
install_packages()
16+
{
17+
sudo dnf install "${packages[@]}"
18+
}
19+
20+
stop_services()
21+
{
22+
cat >benchmark.target <<EOF
23+
[Unit]
24+
Description=Minimal Benchmarking mode with SSH/getty
25+
Requires=sysinit.target getty.target network.target sshd.service
26+
After=sysinit.target
27+
AllowIsolate=Yes
28+
EOF
29+
sudo mv benchmark.target /run/systemd/system/
30+
sudo systemctl daemon-reload
31+
sudo systemctl isolate benchmark.target
32+
sudo systemctl list-units --state=running --no-pager
33+
}
34+
35+
start_services()
36+
{
37+
sudo systemctl isolate multi-user.target
38+
}
39+
40+
set_cpu()
41+
{
42+
# Prevent frequency-scaling
43+
sudo cpupower frequency-set -g performance
44+
45+
# Prevent entering deep C-states to reduce latency and avoid menu governor bugs (there was one recently)
46+
# Although if we completely disable C1 then we are going to hit RAPL power limits immediately,
47+
# which results in a 30% perf hit on single-stream iperf
48+
# Disable just deeper C states, which avoids the big perf hit (although now we're exposed to menu governor bugs)
49+
sudo cpupower idle-set -D 2
50+
51+
# Disable Turbo Boost for more stable measurements
52+
# cpupower set --turbo-boost 0 doesn't work when the driver is intel_pstate instead of cpufreq
53+
sudo sh -c 'echo 1 >/sys/devices/system/cpu/intel_pstate/no_turbo'
54+
55+
# AMD
56+
sudo cpupower set -m passive
57+
sudo cpupower set --turbo-boost 0
58+
59+
sudo cpupower -c all set --perf-bias 0
60+
61+
# Bring system into a consistent state
62+
sudo tuned-adm profile network-throughput
63+
64+
# we could use tuned-adm verify, but there is a bug where it tries to read non-existent hung_task_timeout_secs
65+
# that only fails on network-latency, not network-throughput though
66+
sudo tuned-adm verify
67+
}
68+
69+
describe()
70+
{
71+
# Check all the settings we've changed above
72+
73+
# List running processes
74+
set -x
75+
sudo systemctl list-units --state=running
76+
LIBPROC_HIDE_KERNEL=1 ps -ejH
77+
78+
sudo cpupower -c all frequency-info
79+
sudo cpupower -c all idle-info
80+
sudo cpupower -c all info
81+
sudo cat /sys/devices/system/cpu/intel_pstate/no_turbo
82+
sudo tuned-adm profile_info
83+
sudo ss -neopatium
84+
85+
lscpu
86+
sudo turbostat true
87+
sudo lspci -k
88+
sudo ip a
89+
90+
for P in /sys/class/net/*; do
91+
IFACE=$(basename "${P}")
92+
sudo ethtool "${IFACE}"
93+
sudo ethtool -k "${IFACE}"
94+
done
95+
96+
sudo sos report --batch
97+
98+
set +x
99+
}
100+
101+
open_ports()
102+
{
103+
# iperf3, nuttcp
104+
for P in 5201 5000 5101 9080 9443 9444 9445 9446 10443 10444 10445 10446 20443 20444 20445 20446; do
105+
sudo firewall-cmd --add-port "${P}/tcp"
106+
done
107+
}
108+
109+
iperf3_server()
110+
{
111+
iperf3 -s
112+
}
113+
114+
iperf3_client()
115+
{
116+
# TODO: autodetect IP
117+
iperf3 --json --logfile iperf3.json -c 10.70.58.43
118+
iperf3 --json --logfile iperf3_2.json -c 10.70.58.43 -P2
119+
iperf3 --json --logfile iperf3_4.json -c 10.70.58.43 -P4
120+
}

‎scripts/stunnel-perf/launch.sh

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/bin/sh
2+
set -eu
3+
4+
STUNNEL=${STUNNEL-stunnel}
5+
6+
# Use a single, fixed cipher for consistent tests across different tools
7+
TLS_CIPHERS=ECDHE-RSA-AES256-GCM-SHA384
8+
TLS_PROTO=TLSv1.2
9+
10+
# Use non-root ports
11+
NGINX_PORT=9080
12+
NGINX_TLS_PORT=9443
13+
STUNNEL_TLS_PORT=9444
14+
HITCH_TLS_PORT=9445
15+
SOCAT_TLS_PORT=9446
16+
17+
REPEAT=${REPEAT-30}
18+
TEST_SIZE=${TEST_SIZE-1GiB}
19+
20+
PEM=stunnel.pem
21+
TESTDIR="${PWD}/tmp"
22+
RAMDIR=/dev/shm/stunnel-test
23+
mkdir -p "${RAMDIR}" "${TESTDIR}"
24+
25+
cd "${TESTDIR}"
26+
27+
test -f "${PEM}" || openssl req -new -subj "/C=GB/CN=foo.example.com" -days 10 -x509 -noenc -batch -out "${PEM}" -keyout "${PEM}" -noenc -batch
28+
29+
cat >stunnel.conf <<EOF
30+
fips=no
31+
socket = r:TCP_NODELAY=1
32+
socket = a:TCP_NODELAY=1
33+
socket = l:TCP_NODELAY=1
34+
socket = r:SO_KEEPALIVE=1
35+
socket = a:SO_KEEPALIVE=1
36+
37+
[test]
38+
accept = :::${STUNNEL_TLS_PORT}
39+
cert = ${PEM}
40+
connect = ${NGINX_PORT}
41+
ciphers = ${TLS_CIPHERS}
42+
options = CIPHER_SERVER_PREFERENCE
43+
sslVersion = ${TLS_PROTO}
44+
EOF
45+
46+
truncate --size "${TEST_SIZE}" "${RAMDIR}/${TEST_SIZE}"
47+
48+
cat >nginx.conf <<EOF
49+
error_log stderr error;
50+
pid ${TESTDIR}/nginx.pid;
51+
daemon on;
52+
events { }
53+
http {
54+
client_body_temp_path "${TESTDIR}/nginx-client_body";
55+
proxy_temp_path "${TESTDIR}/nginx-proxy";
56+
fastcgi_temp_path "${TESTDIR}/nginx-fcgi";
57+
uwsgi_temp_path ${TESTDIR}/nginx-uwsgi";
58+
scgi_temp_path ${TESTDIR}/nginx-scgi";
59+
access_log off;
60+
tcp_nodelay on;
61+
sendfile on;
62+
server {
63+
gzip off;
64+
listen "${NGINX_PORT}" reuseport;
65+
listen "${NGINX_TLS_PORT}" reuseport ssl;
66+
ssl_certificate "${PEM}";
67+
ssl_certificate_key "${PEM}";
68+
ssl_protocols "${TLS_PROTO}";
69+
ssl_ciphers "${TLS_CIPHERS}";
70+
root "${RAMDIR}";
71+
location = /hello {
72+
default_type text/plain;
73+
return 200 "Hello";
74+
}
75+
}
76+
}
77+
EOF
78+
79+
cat >hitch.conf <<EOF
80+
backend="[127.0.0.1]:${NGINX_PORT}"
81+
ciphers="${TLS_CIPHERS}"
82+
daemon=on
83+
frontend={
84+
host="*"
85+
port="${HITCH_TLS_PORT}"
86+
pem-file="${PEM}"
87+
prefer-server-ciphers=on
88+
tls-protos=${TLS_PROTO}
89+
}
90+
EOF
91+
92+
killall -v -9 stunnel nginx hitch socat || :
93+
94+
# Start servers
95+
nginx -c "${TESTDIR}/nginx.conf"
96+
socat -b65536 "OPENSSL-LISTEN:${SOCAT_TLS_PORT},fork,reuseaddr,cert=${PEM},verify=0,cipher=${TLS_CIPHERS},compress=none,reuseport" "TCP4:127.0.0.1:${NGINX_PORT}" &
97+
${STUNNEL} "${TESTDIR}/stunnel.conf"
98+
hitch --config "${TESTDIR}/hitch.conf"
99+
100+
DEST=${DEST:-127.0.0.1}
101+
102+
# Start clients (they could also use a pipe, for simplicity use a local port)
103+
for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}; do
104+
PORT2=$(( PORT + 10000 ))
105+
cat >"stunnel-client-${PORT}.conf" <<EOF
106+
client=yes
107+
foreground=no
108+
socket=r:TCP_NODELAY=1
109+
socket=r:SO_KEEPALIVE=1
110+
socket=a:SO_KEEPALIVE=1
111+
fips=no
112+
connect=${DEST}:${PORT}
113+
sslVersion=${TLS_PROTO}
114+
ciphers=${TLS_CIPHERS}
115+
[client-proxy]
116+
accept=127.0.0.1:${PORT2}
117+
EOF
118+
"${STUNNEL}" "${TESTDIR}/stunnel-client-${PORT}.conf"
119+
PORT3=$(( PORT + 20000 ))
120+
socat -b65536 "TCP4-LISTEN:${PORT3},fork,bind=127.0.0.1,reuseport" "OPENSSL:${DEST}:${PORT},cipher=${TLS_CIPHERS},verify=0"&
121+
done
122+
123+
[ "${SERVER-0}" -eq 1 ] && exit 0
124+
125+
measure()
126+
{
127+
echo
128+
echo "${4}: curl --> ${1} ${2} ${3} ..."
129+
for _ in $(seq "${REPEAT}"); do
130+
curl -k -o /dev/null "${2}/${TEST_SIZE}" --write-out "%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n" --silent >>"${4}.csv"
131+
done
132+
133+
echo "${4}: curl --> ${1} ${2} ${3}" >&2
134+
# Calculate data transfer time
135+
python3 >"${4}.dat" <<EOF
136+
import csv
137+
import sys
138+
from statistics import median
139+
140+
def print_speed(msg:str, amount:float, time:float) -> None:
141+
speed = amount * 8 / time / 1e9
142+
print(f"{msg}: {speed} Gbit/s", file=sys.stderr)
143+
144+
with open("${4}.csv", "r") as csvfile:
145+
rows = list(row for row in csv.reader(csvfile))
146+
amount=None
147+
for row in rows:
148+
if (row[2] != '200'):
149+
print(row, file=sys.stderr,flush=True)
150+
sys.exit(1)
151+
if amount is None:
152+
amount=row[3]
153+
if (row[3] != amount):
154+
print("Amount transferred is different", row, file=sys.stderr, flush=True)
155+
sys.exit(1)
156+
times = list(float(row[6]) - float(row[5]) for row in rows)
157+
for time in times:
158+
print(time)
159+
print_speed("min", float(amount), min(times))
160+
print_speed("median", float(amount), median(times))
161+
print_speed("max", float(amount), max(times))
162+
sys.exit(0)
163+
EOF
164+
}
165+
166+
measure "nginx" "http://${DEST}:${NGINX_PORT}" "" "plain"
167+
168+
NGINX_URL="http://127.0.0.1:${NGINX_PORT}"
169+
170+
measure "nginx-ssl" "https://${DEST}:${NGINX_TLS_PORT}" "" "nginx-ssl"
171+
172+
measure "stunnel" "https://${DEST}:${STUNNEL_TLS_PORT}" "-> nginx ${NGINX_URL}" "stunnel_server"
173+
measure "hitch" "https://${DEST}:${HITCH_TLS_PORT}" "-> nginx ${NGINX_URL}" "hitch_server"
174+
measure "socat" "https://${DEST}:${SOCAT_TLS_PORT}" "-> nginx ${NGINX_URL}" "socat_server"
175+
176+
measure "stunnel(client)" "http://127.0.0.1:1${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_stunnel"
177+
measure "stunnel(client)" "http://127.0.0.1:1${NGINX_TLS_PORT}" "-> nginx https://${DEST}:${NGINX_TLS_PORT}" "stunnel_client_nginx"
178+
measure "stunnel(client)" "http://127.0.0.1:1${HITCH_TLS_PORT}" "-> hitch https://${DEST}:${HITCH_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_hitch"
179+
measure "stunnel(client)" "http://127.0.0.1:1${SOCAT_TLS_PORT}" "-> socat https://${DEST}:${SOCAT_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_socat"
180+
181+
measure "socat(client)" "http://127.0.0.1:2${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}" "socat_client_stunnel"
182+
183+
echo "DONE"
184+
exit 0
185+

0 commit comments

Comments
 (0)