Skip to content

Commit 4d23e2c

Browse files
committed
profile,integration: add bounded WireGuard benchmarks
Add reusable Go and one-tunnel P1/P4/P8 integration benchmarks without changing production behavior. Bound Go benchmark process groups and every iperf run so a stalled fixture cannot consume the VM indefinitely, including the process-exit race at the timeout boundary. Validate a full window of unique packets outside timed metrics so duplicate, missing, changed, or excess deliveries fail. Make fake-fabric shutdown interrupt blocked sends and cover both correctness regressions directly.
1 parent 1afa3fd commit 4d23e2c

5 files changed

Lines changed: 985 additions & 3 deletions

File tree

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env python3
2+
"""Run a benchmark command in its own process group with a hard timeout."""
3+
4+
import os
5+
import signal
6+
import subprocess
7+
import sys
8+
9+
10+
def run_bounded(seconds: float, command: list[str]) -> int:
11+
process = subprocess.Popen(command, start_new_session=True)
12+
try:
13+
return process.wait(timeout=seconds)
14+
except subprocess.TimeoutExpired:
15+
try:
16+
os.killpg(process.pid, signal.SIGQUIT)
17+
except ProcessLookupError:
18+
pass
19+
try:
20+
process.wait(timeout=5)
21+
except subprocess.TimeoutExpired:
22+
pass
23+
try:
24+
os.killpg(process.pid, signal.SIGKILL)
25+
except ProcessLookupError:
26+
pass
27+
process.wait()
28+
print(f"benchmark timed out after {seconds:g}s", file=sys.stderr)
29+
return 124
30+
31+
32+
def main() -> int:
33+
if len(sys.argv) < 3:
34+
raise SystemExit(f"usage: {sys.argv[0]} SECONDS COMMAND [ARG ...]")
35+
seconds = float(sys.argv[1])
36+
command = sys.argv[2:]
37+
if command[0] == "--":
38+
command = command[1:]
39+
if not command:
40+
raise SystemExit("missing command")
41+
return run_bounded(seconds, command)
42+
43+
44+
if __name__ == "__main__":
45+
raise SystemExit(main())
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env python3
2+
3+
import subprocess
4+
import unittest
5+
from unittest import mock
6+
7+
import run_bounded_benchmark
8+
9+
10+
class RunBoundedBenchmarkTest(unittest.TestCase):
11+
def test_process_group_exit_at_deadline_returns_timeout(self) -> None:
12+
process = mock.Mock(pid=1234)
13+
process.wait.side_effect = [
14+
subprocess.TimeoutExpired(["benchmark"], 1),
15+
0,
16+
0,
17+
]
18+
with (
19+
mock.patch.object(run_bounded_benchmark.subprocess, "Popen", return_value=process) as popen,
20+
mock.patch.object(run_bounded_benchmark.os, "killpg", side_effect=ProcessLookupError) as killpg,
21+
):
22+
status = run_bounded_benchmark.run_bounded(1, ["benchmark"])
23+
24+
self.assertEqual(status, 124)
25+
popen.assert_called_once_with(["benchmark"], start_new_session=True)
26+
self.assertEqual(
27+
process.wait.call_args_list,
28+
[mock.call(timeout=1), mock.call(timeout=5), mock.call()],
29+
)
30+
self.assertEqual(killpg.call_count, 2)
31+
32+
33+
if __name__ == "__main__":
34+
unittest.main()

‎integration/wireguard_benchmark.sh‎

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
#!/usr/bin/env bash
2+
# Measure TCP goodput through two complete Tailmix WireGuard stacks in isolated
3+
# Linux network namespaces. Results describe this machine and are not portable
4+
# performance assertions.
5+
set -euo pipefail
6+
7+
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
8+
work_dir=$(mktemp -d "${TMPDIR:-/tmp}/tailmix-wireguard-benchmark.XXXXXX")
9+
run_id=$$
10+
ns_a="tmwgba${run_id}"
11+
ns_b="tmwgbb${run_id}"
12+
veth_a="tmwbva${run_id}"
13+
veth_b="tmwbvb${run_id}"
14+
background_pids=()
15+
concurrencies=(1 4 8)
16+
iperf3_bin=
17+
18+
cleanup() {
19+
local rc=$?
20+
trap - EXIT INT TERM
21+
set +e
22+
for pid in "${background_pids[@]}"; do
23+
sudo kill "$pid" 2>/dev/null || true
24+
done
25+
for ns in "$ns_a" "$ns_b"; do
26+
mapfile -t pids < <(sudo ip netns pids "$ns" 2>/dev/null)
27+
if ((${#pids[@]})); then
28+
sudo kill "${pids[@]}" 2>/dev/null || true
29+
fi
30+
sudo ip netns del "$ns" 2>/dev/null || true
31+
done
32+
sudo rm -rf "$work_dir"
33+
exit "$rc"
34+
}
35+
trap cleanup EXIT INT TERM
36+
37+
fail() {
38+
printf 'FAIL: %s\n' "$*" >&2
39+
exit 1
40+
}
41+
42+
require_command() {
43+
command -v "$1" >/dev/null 2>&1 || fail "required command is unavailable: $1"
44+
}
45+
46+
for command in awk date getconf git go grep head ip iperf3 mktemp python3 seq ss sudo timeout uname; do
47+
require_command "$command"
48+
done
49+
[[ $(uname -s) == Linux ]] || fail "Linux is required"
50+
sudo -n true 2>/dev/null || fail "passwordless sudo is required"
51+
[[ -c /dev/net/tun ]] || fail "/dev/net/tun is unavailable"
52+
iperf3_bin=$(command -v iperf3)
53+
[[ $iperf3_bin == /* ]] || fail "iperf3 did not resolve to an absolute path"
54+
55+
mkdir -p "$work_dir/bin" "$work_dir/a/run" "$work_dir/b/run" "$work_dir/results"
56+
(
57+
cd "$repo_root"
58+
go build -o "$work_dir/bin/tailmixd" ./cmd/tailmixd
59+
go build -o "$work_dir/bin/tailmix" ./cmd/tailmix
60+
)
61+
62+
cat >"$work_dir/keygen.go" <<'GO'
63+
package main
64+
65+
import (
66+
"fmt"
67+
68+
"github.com/maisem/tailmix/wireguardcfg"
69+
)
70+
71+
func main() {
72+
for range 2 {
73+
private, err := wireguardcfg.GeneratePrivateKey()
74+
if err != nil {
75+
panic(err)
76+
}
77+
public, err := private.Public()
78+
if err != nil {
79+
panic(err)
80+
}
81+
fmt.Printf("%s %s\n", private.String(), public.String())
82+
}
83+
}
84+
GO
85+
mapfile -t key_pairs < <(cd "$repo_root" && go run "$work_dir/keygen.go")
86+
((${#key_pairs[@]} == 2)) || fail "key generation returned an unexpected result"
87+
read -r private_a public_a <<<"${key_pairs[0]}"
88+
read -r private_b public_b <<<"${key_pairs[1]}"
89+
printf '%s\n' "$private_a" >"$work_dir/a.key"
90+
printf '%s\n' "$private_b" >"$work_dir/b.key"
91+
chmod 0600 "$work_dir/a.key" "$work_dir/b.key"
92+
93+
sudo ip netns add "$ns_a"
94+
sudo ip netns add "$ns_b"
95+
sudo ip link add "$veth_a" type veth peer name "$veth_b"
96+
sudo ip link set "$veth_a" netns "$ns_a"
97+
sudo ip link set "$veth_b" netns "$ns_b"
98+
sudo ip -n "$ns_a" link set lo up
99+
sudo ip -n "$ns_b" link set lo up
100+
sudo ip -n "$ns_a" link set "$veth_a" name underlay0
101+
sudo ip -n "$ns_b" link set "$veth_b" name underlay0
102+
sudo ip -n "$ns_a" address add 192.0.2.1/30 dev underlay0
103+
sudo ip -n "$ns_b" address add 192.0.2.2/30 dev underlay0
104+
sudo ip -n "$ns_a" link set underlay0 up
105+
sudo ip -n "$ns_b" link set underlay0 up
106+
107+
sudo ip netns exec "$ns_a" "$work_dir/bin/tailmixd" \
108+
-auto-update=false \
109+
-state "$work_dir/a/state.json" \
110+
-socket-dir "$work_dir/a/run" \
111+
-tun-name tailmix0 >"$work_dir/a-daemon.log" 2>&1 &
112+
background_pids+=("$!")
113+
sudo ip netns exec "$ns_b" "$work_dir/bin/tailmixd" \
114+
-auto-update=false \
115+
-state "$work_dir/b/state.json" \
116+
-socket-dir "$work_dir/b/run" \
117+
-tun-name tailmix0 >"$work_dir/b-daemon.log" 2>&1 &
118+
background_pids+=("$!")
119+
120+
wait_for_file() {
121+
local path=$1
122+
for _ in $(seq 1 150); do
123+
[[ -e $path ]] && return 0
124+
sleep 0.1
125+
done
126+
return 1
127+
}
128+
129+
wait_for_file "$work_dir/a/run/tailmixd.sock" || fail "first daemon did not create its control socket"
130+
wait_for_file "$work_dir/b/run/tailmixd.sock" || fail "second daemon did not create its control socket"
131+
132+
write_manifest() {
133+
local side=$1 name self_address peer_name peer_address peer_public endpoint private_file
134+
if [[ $side == a ]]; then
135+
name=alpha
136+
self_address=10.77.0.1
137+
peer_name=beta
138+
peer_address=10.77.0.2
139+
peer_public=$public_b
140+
endpoint=192.0.2.2:51820
141+
private_file=$work_dir/a.key
142+
else
143+
name=beta
144+
self_address=10.77.0.2
145+
peer_name=alpha
146+
peer_address=10.77.0.1
147+
peer_public=$public_a
148+
endpoint=192.0.2.1:51820
149+
private_file=$work_dir/b.key
150+
fi
151+
cat >"$work_dir/$side.yaml" <<YAML
152+
version: 1
153+
name: $name
154+
dnsSuffix: $name.test
155+
addresses: [$self_address]
156+
listenPort: 51820
157+
privateKeyFile: $private_file
158+
packetFilter:
159+
grants:
160+
- src: [peer:$peer_name]
161+
dst: [self]
162+
ip: [tcp:5201]
163+
peers:
164+
- name: $peer_name
165+
publicKey: $peer_public
166+
endpoint: $endpoint
167+
keepalive: 1s
168+
addresses: [$peer_address]
169+
YAML
170+
}
171+
172+
apply_profile() {
173+
local side=$1
174+
write_manifest "$side"
175+
sudo env TAILMIX_SOCKET_DIR="$work_dir/$side/run" \
176+
"$work_dir/bin/tailmix" wireguard apply --file "$work_dir/$side.yaml" --json \
177+
>"$work_dir/$side-profile.json"
178+
}
179+
180+
apply_profile a
181+
apply_profile b
182+
183+
effective_b=$(python3 - "$work_dir/a-profile.json" <<'PY'
184+
import json
185+
import sys
186+
187+
with open(sys.argv[1], encoding="utf-8") as source:
188+
profile = json.load(source)
189+
print(profile["peers"][0]["effectiveAddresses"][0])
190+
PY
191+
)
192+
[[ -n $effective_b ]] || fail "first profile did not report an effective peer address"
193+
194+
sudo ip netns exec "$ns_b" "$iperf3_bin" -s >"$work_dir/iperf3-server.log" 2>&1 &
195+
background_pids+=("$!")
196+
197+
server_ready=0
198+
for _ in $(seq 1 100); do
199+
if sudo ip netns exec "$ns_b" ss -ltn | grep -F ':5201 ' >/dev/null; then
200+
server_ready=1
201+
break
202+
fi
203+
sleep 0.1
204+
done
205+
((server_ready == 1)) || fail "iperf3 server did not listen"
206+
207+
# Establish the WireGuard session and validate the complete data path before
208+
# starting the measured repetitions. The output is intentionally discarded.
209+
sudo ip netns exec "$ns_a" timeout --signal=TERM --kill-after=5s 15s "$iperf3_bin" \
210+
-c "$effective_b" -P 1 -t 1 -J >"$work_dir/iperf3-warmup.json"
211+
212+
run_iperf() {
213+
local concurrency=$1 direction=$2 run=$3
214+
local -a command=(timeout --signal=TERM --kill-after=5s 30s "$iperf3_bin" -c "$effective_b" -P "$concurrency" -t 12 -O 2 -J)
215+
if [[ $direction == reverse ]]; then
216+
command+=(-R)
217+
fi
218+
sudo ip netns exec "$ns_a" "${command[@]}" >"$work_dir/results/p$concurrency-$direction-$run.json"
219+
}
220+
221+
for concurrency in "${concurrencies[@]}"; do
222+
for direction in normal reverse; do
223+
for run in 1 2 3; do
224+
run_iperf "$concurrency" "$direction" "$run"
225+
done
226+
done
227+
done
228+
229+
commit=$(git -C "$repo_root" rev-parse HEAD)
230+
if [[ -n $(git -C "$repo_root" status --porcelain) ]]; then
231+
worktree_state=dirty
232+
else
233+
worktree_state=clean
234+
fi
235+
cpu_model=$(awk -F: '/model name/ {sub(/^[ \t]+/, "", $2); print $2; exit}' /proc/cpuinfo)
236+
logical_cpus=$(getconf _NPROCESSORS_ONLN)
237+
238+
printf 'Tailmix WireGuard end-to-end benchmark\n'
239+
printf 'date_utc: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
240+
printf 'commit: %s (%s)\n' "$commit" "$worktree_state"
241+
printf 'go: %s\n' "$(go version)"
242+
printf 'iperf3: %s [%s]\n' "$($iperf3_bin --version 2>&1 | head -1)" "$iperf3_bin"
243+
printf 'kernel: %s\n' "$(uname -r)"
244+
printf 'cpu: %s\n' "$cpu_model"
245+
printf 'logical_cpus: %s\n' "$logical_cpus"
246+
printf 'parameters: TCP, P1/P4/P8 through one tunnel, normal+reverse, 3 repetitions/concurrency/direction, 12s/run, omit first 2s, 30s hard timeout/run\n'
247+
248+
python3 - "$work_dir/results" <<'PY'
249+
import json
250+
import pathlib
251+
import statistics
252+
import sys
253+
254+
results_dir = pathlib.Path(sys.argv[1])
255+
for concurrency in (1, 4, 8):
256+
for direction in ("normal", "reverse"):
257+
throughputs = []
258+
print(f"P{concurrency} {direction} runs:")
259+
for run in range(1, 4):
260+
path = results_dir / f"p{concurrency}-{direction}-{run}.json"
261+
with path.open(encoding="utf-8") as source:
262+
result = json.load(source)
263+
error = result.get("error")
264+
if error:
265+
raise SystemExit(
266+
f"iperf3 P{concurrency} {direction} run {run} failed: {error}"
267+
)
268+
end = result["end"]
269+
bits_per_second = float(end["sum_received"]["bits_per_second"])
270+
retransmits = int(end.get("sum_sent", {}).get("retransmits", 0))
271+
throughputs.append(bits_per_second)
272+
print(
273+
f" run {run}: {bits_per_second / 1_000_000_000:.3f} Gbit/s, "
274+
f"retransmits={retransmits}"
275+
)
276+
minimum = min(throughputs)
277+
maximum = max(throughputs)
278+
median = statistics.median(throughputs)
279+
unstable = maximum - minimum > 0.10 * median
280+
print(
281+
f" summary: median={median / 1_000_000_000:.3f} Gbit/s "
282+
f"min={minimum / 1_000_000_000:.3f} Gbit/s "
283+
f"max={maximum / 1_000_000_000:.3f} Gbit/s "
284+
f"unstable={'yes' if unstable else 'no'}"
285+
)
286+
PY

0 commit comments

Comments
 (0)