Skip to content

Commit ff8da3a

Browse files
edwintorokedwintorok
committed
nginx/perf
1 parent d27a118 commit ff8da3a

File tree

3 files changed

+297
-0
lines changed

3 files changed

+297
-0
lines changed

‎scripts/stunnel-perf/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tmp/

‎scripts/stunnel-perf/benchmark.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/bash
2+
# Set a machine into benchmarking mode.
3+
#
4+
# For native Linux, and Intel-only for now (AMD has a different way of disabling turbo)
5+
#
6+
# References: https://llvm.org/docs/Benchmarking.html
7+
8+
# This is for Fedora 42, adapt for your own OS
9+
declare -a packages=(
10+
"cpupower" "tuned" "ss" "turbostat" "lscpu" "lspci" "ip" "ethtool" "jq" "nginx" "hitch" "stunnel" "iperf3" "nuttcp" "httperf" "git" "hyperfine" "varnish" "socat" "curl"
11+
)
12+
13+
# httperf --server $OTHER_IP --port 8080 --num-calls 3000 --num-conns=1 --uri /hello
14+
15+
install_packages()
16+
{
17+
sudo dnf install "${packages[@]}"
18+
}
19+
20+
stop_services()
21+
{
22+
sudo cp /lib/systemd/system/rescue.target /lib/systemd/system/benchmark.target
23+
sudo systemctl add-wants benchmark.target sshd.service
24+
sudo systemctl isolate benchmark.target
25+
sudo systemctl list-units --state=running
26+
}
27+
28+
start_services()
29+
{
30+
sudo systemctl isolate multi-user.target
31+
}
32+
33+
set_cpu()
34+
{
35+
# Prevent frequency-scaling
36+
sudo cpupower frequency-set -g performance
37+
38+
# Prevent entering deep C-states to reduce latency and avoid menu governor bugs (there was one recently)
39+
# Although if we completely disable C1 then we are going to hit RAPL power limits immediately,
40+
# which results in a 30% perf hit on single-stream iperf
41+
# Disable just deeper C states, which avoids the big perf hit (although now we're exposed to menu governor bugs)
42+
sudo cpupower idle-set -D 2
43+
44+
# Disable Turbo Boost for more stable measurements
45+
# cpupower set --turbo-boost 0 doesn't work when the driver is intel_pstate instead of cpufreq
46+
sudo sh -c 'echo 1 >/sys/devices/system/cpu/intel_pstate/no_turbo'
47+
48+
# AMD
49+
sudo cpupower set -m passive
50+
sudo cpupower set --turbo-boost 0
51+
52+
sudo cpupower -c all set --perf-bias 0
53+
54+
# Bring system into a consistent state
55+
sudo tuned-adm profile network-throughput
56+
57+
# we could use tuned-adm verify, but there is a bug where it tries to read non-existent hung_task_timeout_secs
58+
# that only fails on network-latency, not network-throughput though
59+
sudo tuned-adm verify
60+
}
61+
62+
describe()
63+
{
64+
# Check all the settings we've changed above
65+
66+
# List running processes
67+
set -x
68+
sudo systemctl list-units --state=running
69+
LIBPROC_HIDE_KERNEL=1 ps -ejH
70+
71+
sudo cpupower -c all frequency-info
72+
sudo cpupower -c all idle-info
73+
sudo cpupower -c all info
74+
sudo cat /sys/devices/system/cpu/intel_pstate/no_turbo
75+
sudo tuned-adm profile_info
76+
sudo ss -neopatium
77+
78+
lscpu
79+
sudo turbostat true
80+
sudo lspci -k
81+
sudo ip a
82+
83+
for P in /sys/class/net/*; do
84+
IFACE=$(basename "${P}")
85+
sudo ethtool "${IFACE}"
86+
sudo ethtool -k "${IFACE}"
87+
done
88+
89+
sudo sos report --batch
90+
91+
set +x
92+
}
93+
94+
open_ports()
95+
{
96+
# iperf3, nuttcp
97+
for P in 5201 5000 5101 9080 9443 9444 9445 9446 10443 10444 10445 10446 20443 20444 20445 20446; do
98+
sudo firewall-cmd --add-port "${P}/tcp"
99+
done
100+
}
101+
102+
iperf3_server()
103+
{
104+
iperf3 -s
105+
}
106+
107+
iperf3_client()
108+
{
109+
# TODO: autodetect IP
110+
iperf3 --json --logfile iperf3.json -c 10.70.58.43
111+
iperf3 --json --logfile iperf3_2.json -c 10.70.58.43 -P2
112+
iperf3 --json --logfile iperf3_4.json -c 10.70.58.43 -P4
113+
}

‎scripts/stunnel-perf/launch.sh

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
#!/bin/sh
2+
set -eu
3+
4+
# Use a single, fixed cipher for consistent tests across different tools
5+
TLS_CIPHERS=ECDHE-RSA-AES256-GCM-SHA384
6+
TLS_PROTO=TLSv1.2
7+
8+
# Use non-root ports
9+
NGINX_PORT=9080
10+
NGINX_TLS_PORT=9443
11+
STUNNEL_TLS_PORT=9444
12+
HITCH_TLS_PORT=9445
13+
SOCAT_TLS_PORT=9446
14+
15+
REPEAT=${REPEAT-30}
16+
TEST_SIZE=${TEST_SIZE-1GiB}
17+
18+
PEM=stunnel.pem
19+
TESTDIR="${PWD}/tmp"
20+
RAMDIR=/dev/shm/stunnel-test
21+
mkdir -p "${RAMDIR}" "${TESTDIR}"
22+
23+
cd "${TESTDIR}"
24+
25+
test -f "${PEM}" || openssl req -new -subj "/C=GB/CN=foo.example.com" -days 10 -x509 -noenc -batch -out "${PEM}" -keyout "${PEM}" -noenc -batch
26+
27+
cat >stunnel.conf <<EOF
28+
fips=no
29+
socket = r:TCP_NODELAY=1
30+
socket = a:TCP_NODELAY=1
31+
socket = l:TCP_NODELAY=1
32+
socket = r:SO_KEEPALIVE=1
33+
socket = a:SO_KEEPALIVE=1
34+
35+
[test]
36+
accept = :::${STUNNEL_TLS_PORT}
37+
cert = ${PEM}
38+
connect = ${NGINX_PORT}
39+
ciphers = ${TLS_CIPHERS}
40+
options = CIPHER_SERVER_PREFERENCE
41+
sslVersion = ${TLS_PROTO}
42+
EOF
43+
44+
truncate --size "${TEST_SIZE}" "${RAMDIR}/${TEST_SIZE}"
45+
46+
cat >nginx.conf <<EOF
47+
error_log stderr error;
48+
pid ${TESTDIR}/nginx.pid;
49+
daemon on;
50+
events { }
51+
http {
52+
client_body_temp_path "${TESTDIR}/nginx-client_body";
53+
proxy_temp_path "${TESTDIR}/nginx-proxy";
54+
fastcgi_temp_path "${TESTDIR}/nginx-fcgi";
55+
uwsgi_temp_path ${TESTDIR}/nginx-uwsgi";
56+
scgi_temp_path ${TESTDIR}/nginx-scgi";
57+
access_log off;
58+
tcp_nodelay on;
59+
sendfile on;
60+
server {
61+
gzip off;
62+
listen "${NGINX_PORT}" reuseport;
63+
listen "${NGINX_TLS_PORT}" reuseport ssl;
64+
ssl_certificate "${PEM}";
65+
ssl_certificate_key "${PEM}";
66+
ssl_protocols "${TLS_PROTO}";
67+
ssl_ciphers "${TLS_CIPHERS}";
68+
root "${RAMDIR}";
69+
location = /hello {
70+
default_type text/plain;
71+
return 200 "Hello";
72+
}
73+
}
74+
}
75+
EOF
76+
77+
cat >hitch.conf <<EOF
78+
backend="[127.0.0.1]:${NGINX_PORT}"
79+
ciphers="${TLS_CIPHERS}"
80+
daemon=on
81+
frontend={
82+
host="*"
83+
port="${HITCH_TLS_PORT}"
84+
pem-file="${PEM}"
85+
prefer-server-ciphers=on
86+
tls-protos=${TLS_PROTO}
87+
}
88+
EOF
89+
90+
killall -v -9 stunnel nginx hitch socat || :
91+
92+
# Start servers
93+
nginx -c "${TESTDIR}/nginx.conf"
94+
socat -b65536 "OPENSSL-LISTEN:${SOCAT_TLS_PORT},fork,reuseaddr,cert=${PEM},verify=0,cipher=${TLS_CIPHERS},compress=none,reuseport" "TCP4:127.0.0.1:${NGINX_PORT}" &
95+
stunnel "${TESTDIR}/stunnel.conf"
96+
hitch --config "${TESTDIR}/hitch.conf"
97+
98+
DEST=${DEST:-127.0.0.1}
99+
100+
# Start clients (they could also use a pipe, for simplicity use a local port)
101+
for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}; do
102+
PORT2=$(( PORT + 10000 ))
103+
cat >"stunnel-client-${PORT}.conf" <<EOF
104+
client=yes
105+
foreground=no
106+
socket=r:TCP_NODELAY=1
107+
socket=r:SO_KEEPALIVE=1
108+
socket=a:SO_KEEPALIVE=1
109+
fips=no
110+
connect=${DEST}:${PORT}
111+
sslVersion=${TLS_PROTO}
112+
ciphers=${TLS_CIPHERS}
113+
[client-proxy]
114+
accept=127.0.0.1:${PORT2}
115+
EOF
116+
stunnel "${TESTDIR}/stunnel-client-${PORT}.conf"
117+
PORT3=$(( PORT + 20000 ))
118+
socat -b65536 "TCP4-LISTEN:${PORT3},fork,bind=127.0.0.1,reuseport" "OPENSSL:${DEST}:${PORT},cipher=${TLS_CIPHERS},verify=0"&
119+
done
120+
121+
[ "${SERVER-0}" -eq 1 ] && exit 0
122+
123+
measure()
124+
{
125+
echo
126+
echo "${4}: curl --> ${1} ${2} ${3} ..."
127+
for _ in $(seq "${REPEAT}"); do
128+
curl -k -o /dev/null "${2}/${TEST_SIZE}" --write-out "%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n" --silent >>"${4}.csv"
129+
done
130+
131+
echo "${4}: curl --> ${1} ${2} ${3}" >&2
132+
# Calculate data transfer time
133+
python3 >"${4}.dat" <<EOF
134+
import csv
135+
import sys
136+
from statistics import median
137+
138+
def print_speed(msg:str, amount:float, time:float) -> None:
139+
speed = amount * 8 / time / 1e9
140+
print(f"{msg}: {speed} Gbit/s", file=sys.stderr)
141+
142+
with open("${4}.csv", "r") as csvfile:
143+
rows = list(row for row in csv.reader(csvfile))
144+
amount=None
145+
for row in rows:
146+
if (row[2] != '200'):
147+
print(row, file=sys.stderr,flush=True)
148+
sys.exit(1)
149+
if amount is None:
150+
amount=row[3]
151+
if (row[3] != amount):
152+
print("Amount transferred is different", row, file=sys.stderr, flush=True)
153+
sys.exit(1)
154+
times = list(float(row[6]) - float(row[5]) for row in rows)
155+
for time in times:
156+
print(time)
157+
print_speed("min", float(amount), min(times))
158+
print_speed("median", float(amount), median(times))
159+
print_speed("max", float(amount), max(times))
160+
sys.exit(0)
161+
EOF
162+
}
163+
164+
measure "nginx" "http://${DEST}:${NGINX_PORT}" "" "plain"
165+
166+
NGINX_URL="http://127.0.0.1:${NGINX_PORT}"
167+
168+
measure "nginx-ssl" "https://${DEST}:${NGINX_TLS_PORT}" "" "nginx-ssl"
169+
170+
measure "stunnel" "https://${DEST}:${STUNNEL_TLS_PORT}" "-> nginx ${NGINX_URL}" "stunnel_server"
171+
measure "hitch" "https://${DEST}:${HITCH_TLS_PORT}" "-> nginx ${NGINX_URL}" "hitch_server"
172+
measure "socat" "https://${DEST}:${SOCAT_TLS_PORT}" "-> nginx ${NGINX_URL}" "socat_server"
173+
174+
measure "stunnel(client)" "http://127.0.0.1:1${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_stunnel"
175+
measure "stunnel(client)" "http://127.0.0.1:1${NGINX_TLS_PORT}" "-> nginx https://${DEST}:${NGINX_TLS_PORT}" "stunnel_client_nginx"
176+
measure "stunnel(client)" "http://127.0.0.1:1${HITCH_TLS_PORT}" "-> hitch https://${DEST}:${HITCH_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_hitch"
177+
measure "stunnel(client)" "http://127.0.0.1:1${SOCAT_TLS_PORT}" "-> socat https://${DEST}:${SOCAT_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_socat"
178+
179+
measure "socat(client)" "http://127.0.0.1:2${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}" "socat_client_stunnel"
180+
181+
echo "DONE"
182+
exit 0
183+

0 commit comments

Comments
 (0)