1212 RUST_BACKTRACE : 1
1313 TOOLCHAIN : nightly
1414 RUSTFLAGS : -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
15- PERF_CMD : record -o perf.data -F997 --call-graph fp -g
15+ PERF_OPT : record -F997 --call-graph fp -g
1616
1717jobs :
1818 bench :
2323 shell : bash
2424
2525 steps :
26- - name : Checkout
26+ - name : Checkout neqo
2727 uses : actions/checkout@v4
2828
29+ - name : Checkout msquic
30+ uses : actions/checkout@v4
31+ with :
32+ repository : microsoft/msquic
33+ ref : main
34+ path : msquic
35+ submodules : true
36+
2937 - name : Set PATH
3038 run : echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}"
3139
@@ -38,10 +46,17 @@ jobs:
3846 - name : Fetch and build NSS and NSPR
3947 uses : ./.github/actions/nss
4048
41- - name : Build
49+ - name : Build neqo
4250 run : |
4351 cargo "+$TOOLCHAIN" bench --features bench --no-run
44- cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
52+ cargo "+$TOOLCHAIN" build --release
53+
54+ name : Build msquic
55+ run : |
56+ mkdir -p msquic/build
57+ cd msquic/build
58+ cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
59+ cmake --build .
4560
4661name : Download cached main-branch results
4762 id : criterion-cache
@@ -61,56 +76,107 @@ jobs:
6176 taskset -c 0 nice -n -20 \
6277 cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
6378
64- # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
65- # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
66- - name : Profile cargo bench transfer
67- run : |
68- taskset -c 0 nice -n -20 \
69- cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
70- --bench --exact "Run multiple transfers with varying seeds" --noplot
71-
72- name : Profile client/server transfer
73- run : |
74- { mkdir server; \
75- cd server; \
76- taskset -c 0 nice -n -20 \
77- cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
78- --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
79- mkdir client; \
80- cd client; \
81- time taskset -c 1 nice -n -20 \
82- cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
83- --bin neqo-client -- --output-dir . "https://$HOST:4433/$SIZE"
84- killall -INT neqo-server
85- cd ${{ github.workspace }}
86- [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
79+ # Compare various configurations of neqo against msquic, and gather perf data
80+ # during the hyperfine runs.
81+ - name : Compare neqo and msquic
8782 env :
88- HOST : localhost
89- SIZE : 1073741824 # 1 GB
83+ HOST : 127.0.0.1
84+ PORT : 4433
85+ SIZE : 134217728 # 128 MB
86+ run : |
87+ TMP=$(mktemp -d)
88+ # Make a cert and key for msquic.
89+ openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
90+ # Make a test file for msquic to serve.
91+ truncate -s "$SIZE" "$TMP/$SIZE"
92+ # Define the commands to run for each client and server.
93+ declare -A client_cmd=(
94+ ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
95+ ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
96+ )
97+ declare -A server_cmd=(
98+ ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null"
99+ ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true"
100+ )
101+
102+ # Replace various placeholders in the commands with the actual values.
103+ # Also generate an extension to append to the file name.
104+ function transmogrify {
105+ CMD=$1
106+ local cc=$2
107+ local pacing=$3
108+ if [ "$cc" != "" ]; then
109+ CMD=${CMD//_cc/--cc $cc}
110+ EXT="-$cc"
111+ fi
112+ if [ "$pacing" == "on" ]; then
113+ CMD=${CMD//_pacing/}
114+ EXT="$EXT-pacing"
115+ else
116+ CMD=${CMD//_pacing/--no-pacing}
117+ EXT="$EXT-nopacing"
118+ fi
119+ }
120+
121+ for server in msquic neqo; do
122+ for client in msquic neqo; do
123+ # msquic doesn't let us configure the congestion control or pacing.
124+ if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
125+ cc_opt=("")
126+ pacing_opt=("")
127+ else
128+ cc_opt=("reno" "cubic")
129+ pacing_opt=("on" "")
130+ fi
131+ for cc in "${cc_opt[@]}"; do
132+ for pacing in "${pacing_opt[@]}"; do
133+ # Make a tag string for this test, for the results.
134+ TAG="$client,$server,$cc,$pacing"
135+ echo "Running benchmarks for $TAG" | tee -a comparison.txt
136+ transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
137+ # shellcheck disable=SC2086
138+ taskset -c 0 nice -n -20 \
139+ perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
140+ PID=$!
141+ transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
142+ # shellcheck disable=SC2086
143+ taskset -c 1 nice -n -20 \
144+ perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
145+ hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
146+ tee -a comparison.txt
147+ echo >> comparison.txt
148+ kill $PID
149+ cat step.md >> steps.md
150+ # Sanity check the size of the last retrieved file.
151+ [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
152+ done
153+ done
154+ done
155+ done
156+ # Merge the results tables generated by hyperfine into a single table.
157+ echo "Transfer of $SIZE bytes over loopback." > comparison.md
158+ awk '(!/^\| Command/ || !c++) && (!/^\|:/ || !d++)' < steps.md |\
159+ sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|:/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
160+ rm -r "$TMP"
90161
91162# Re-enable turboboost, hyperthreading and use powersave governor.
92163 - name : Restore machine
93164 run : sudo /root/bin/unprep.sh
94165 if : success() || failure() || cancelled()
95166
96- - name : Convert for profiler.firefox.com
167+ - name : Post-process perf data
97168 run : |
98- perf script -i perf.data -F +pid > transfer.perf &
99- perf script -i client/perf.data -F +pid > client.perf &
100- perf script -i server/perf.data -F +pid > server.perf &
169+ for f in *.perf; do
170+ # Convert for profiler.firefox.com
171+ perf script -i "$f" -F +pid > "$f.fx" &
172+ # Generate perf reports
173+ perf report -i "$f" --no-children --stdio > "$f.txt" &
174+ # Generate flamegraphs
175+ flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}" &
176+ done
101177 wait
102- mv flamegraph.svg transfer.svg
103- mv client/flamegraph.svg client.svg
104- mv server/flamegraph.svg server.svg
105178 rm neqo.svg
106179
107- name : Generate perf reports
108- run : |
109- perf report -i perf.data --no-children --stdio > transfer.perf.txt &
110- perf report -i client/perf.data --no-children --stdio > client.perf.txt &
111- perf report -i server/perf.data --no-children --stdio > server.perf.txt &
112- wait
113-
114180name : Format results as Markdown
115181 id : results
116182 run : |
@@ -132,6 +198,11 @@ jobs:
132198 -e 's/^([a-z0-9].*)$/* **\1**/gi' \
133199 -e 's/(change:[^%]*% )([^%]*%)(.*)/\1**\2**\3/gi' \
134200 >> results.md
201+ {
202+ echo "### Client/server transfer results"
203+ cat comparison.md
204+ } >> results.md
205+ cat results.md > "$GITHUB_STEP_SUMMARY"
135206
136207name : Remember main-branch push URL
137208 if : github.ref == 'refs/heads/main'
@@ -158,6 +229,7 @@ jobs:
158229 path : |
159230 *.svg
160231 *.perf
232+ *.perf.fx
161233 *.txt
162234 results.*
163235 target/criterion*
0 commit comments