From f226e5311861269d9f958f04544456030470c298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Val=C3=A8re=20Plantevin?= Date: Wed, 13 May 2026 15:32:23 -0400 Subject: [PATCH] Results and script to verify netem --- data/loopback/final_table.csv | 13 --- data/two_machine/final_table.csv | 24 ++--- data/two_machine_OLD/final_table.csv | 13 +++ scripts/verify-netem.sh | 135 +++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 25 deletions(-) delete mode 100644 data/loopback/final_table.csv create mode 100644 data/two_machine_OLD/final_table.csv create mode 100755 scripts/verify-netem.sh diff --git a/data/loopback/final_table.csv b/data/loopback/final_table.csv deleted file mode 100644 index f482e87..0000000 --- a/data/loopback/final_table.csv +++ /dev/null @@ -1,13 +0,0 @@ -entities,loss_pct,devices,rate_hz,t1_received,t1_dropped,t1_p50_us,t1_p99_us,t1_p999_us,t2_p99_us,t3_rtt_us,hz,rss_mb -10000,0,1428,100,5001,0,45238.33069752144,48411.85646346627,53098.29148381443,0,8630.865124336387,41257.9,12.0 -10000,1,1428,100,5001,0,45102.819074046034,47662.49174202053,50569.30401137826,0,42281.293387087724,22795.6,15.7 -10000,5,1428,100,5001,0,45030.712264117974,49449.14422462419,58098.785067702745,0,46383.544495318434,15843.7,19.0 -50000,0,7142,100,5001,0,44922.76813908747,48112.63150827307,55742.604812933576,0,9678.86600914163,12142.1,22.0 -50000,1,7142,100,5001,0,44797.160320888564,47757.912114388164,53204.59433455289,0,12706.894847406826,9835.6,26.4 -50000,5,7142,100,5001,0,44662.970225264624,47320.5542524692,50792.29917937587,0,45012.70358112902,8264.4,28.3 -100000,0,14285,100,5002,0,44538.08882376887,47681.5605522887,51477.49656030953,0,9963.662717402913,7115.0,30.1 -100000,1,14285,100,5001,0,44449.10166264818,47405.807955722245,52192.81032823135,0,14046.09958810414,6260.0,34.0 -100000,5,14285,100,5001,0,44369.16524374881,47510.21553980436,52213.691628413864,0,46023.16017305715,5547.8,40.9 -200000,0,28571,100,5001,0,44245.10534556269,47538.73022277268,50934.71690932847,0,9449.338570630583,5012.4,42.6 -200000,1,28571,100,5001,0,44121.392329231494,47159.93757056208,51539.306635015615,0,25644.900728832217,4565.4,44.2 -200000,5,28571,100,5001,0,44033.23772826498,47112.801204945215,50822.78370342773,0,45656.44371211316,4190.2,45.9 diff --git a/data/two_machine/final_table.csv b/data/two_machine/final_table.csv index a919086..f482e87 100644 --- a/data/two_machine/final_table.csv +++ b/data/two_machine/final_table.csv @@ -1,13 +1,13 @@ entities,loss_pct,devices,rate_hz,t1_received,t1_dropped,t1_p50_us,t1_p99_us,t1_p999_us,t2_p99_us,t3_rtt_us,hz,rss_mb -10000,0,2000,100,5002,0,88406.43436980792,92088.01036052403,95215.6914367099,0,0,41812.4,11.4 -10000,1,2000,100,5001,0,88885.12040609193,92051.1825223156,94381.46931014946,0,0,23186.8,14.4 -10000,5,2000,100,5002,0,70213.79159588156,74050.31194954121,79276.73877316424,0,0,16132.9,17.3 -50000,0,10000,100,5001,0,70467.01677831604,74481.05169804857,78660.78554913378,0,0,12318.6,20.0 -50000,1,10000,100,4999,0,70990.40685911797,74854.38952456272,80926.7766369622,0,0,9959.4,23.7 -50000,5,10000,100,5001,0,71389.06834919532,74944.26870901692,79070.8869749687,0,0,8396.9,25.4 -100000,0,20000,100,5000,0,71675.19649834004,75365.13393797085,78976.05881855593,0,0,7224.8,27.1 -100000,1,20000,100,4998,0,72106.54041649138,76718.64382761203,81527.85603693608,0,0,6353.7,30.9 -100000,5,20000,100,4997,0,72453.48380965949,75894.54063901275,78049.6180715917,0,0,5660.0,36.9 -200000,0,40000,100,4992,0,72758.4283780017,77815.82008892013,81936.51611730906,0,0,5062.7,38.7 -200000,1,40000,100,4989,0,73064.65640691575,76841.49191040442,80362.26728642671,0,0,4586.3,40.5 -200000,5,40000,100,3661,0,73313.49903227342,76641.96353006759,78346.77085396706,0,0,4241.5,42.0 +10000,0,1428,100,5001,0,45238.33069752144,48411.85646346627,53098.29148381443,0,8630.865124336387,41257.9,12.0 +10000,1,1428,100,5001,0,45102.819074046034,47662.49174202053,50569.30401137826,0,42281.293387087724,22795.6,15.7 +10000,5,1428,100,5001,0,45030.712264117974,49449.14422462419,58098.785067702745,0,46383.544495318434,15843.7,19.0 +50000,0,7142,100,5001,0,44922.76813908747,48112.63150827307,55742.604812933576,0,9678.86600914163,12142.1,22.0 +50000,1,7142,100,5001,0,44797.160320888564,47757.912114388164,53204.59433455289,0,12706.894847406826,9835.6,26.4 +50000,5,7142,100,5001,0,44662.970225264624,47320.5542524692,50792.29917937587,0,45012.70358112902,8264.4,28.3 +100000,0,14285,100,5002,0,44538.08882376887,47681.5605522887,51477.49656030953,0,9963.662717402913,7115.0,30.1 +100000,1,14285,100,5001,0,44449.10166264818,47405.807955722245,52192.81032823135,0,14046.09958810414,6260.0,34.0 +100000,5,14285,100,5001,0,44369.16524374881,47510.21553980436,52213.691628413864,0,46023.16017305715,5547.8,40.9 +200000,0,28571,100,5001,0,44245.10534556269,47538.73022277268,50934.71690932847,0,9449.338570630583,5012.4,42.6 +200000,1,28571,100,5001,0,44121.392329231494,47159.93757056208,51539.306635015615,0,25644.900728832217,4565.4,44.2 +200000,5,28571,100,5001,0,44033.23772826498,47112.801204945215,50822.78370342773,0,45656.44371211316,4190.2,45.9 diff --git a/data/two_machine_OLD/final_table.csv b/data/two_machine_OLD/final_table.csv new file mode 100644 index 0000000..a919086 --- /dev/null +++ b/data/two_machine_OLD/final_table.csv @@ -0,0 +1,13 @@ +entities,loss_pct,devices,rate_hz,t1_received,t1_dropped,t1_p50_us,t1_p99_us,t1_p999_us,t2_p99_us,t3_rtt_us,hz,rss_mb +10000,0,2000,100,5002,0,88406.43436980792,92088.01036052403,95215.6914367099,0,0,41812.4,11.4 +10000,1,2000,100,5001,0,88885.12040609193,92051.1825223156,94381.46931014946,0,0,23186.8,14.4 +10000,5,2000,100,5002,0,70213.79159588156,74050.31194954121,79276.73877316424,0,0,16132.9,17.3 +50000,0,10000,100,5001,0,70467.01677831604,74481.05169804857,78660.78554913378,0,0,12318.6,20.0 +50000,1,10000,100,4999,0,70990.40685911797,74854.38952456272,80926.7766369622,0,0,9959.4,23.7 +50000,5,10000,100,5001,0,71389.06834919532,74944.26870901692,79070.8869749687,0,0,8396.9,25.4 +100000,0,20000,100,5000,0,71675.19649834004,75365.13393797085,78976.05881855593,0,0,7224.8,27.1 +100000,1,20000,100,4998,0,72106.54041649138,76718.64382761203,81527.85603693608,0,0,6353.7,30.9 +100000,5,20000,100,4997,0,72453.48380965949,75894.54063901275,78049.6180715917,0,0,5660.0,36.9 +200000,0,40000,100,4992,0,72758.4283780017,77815.82008892013,81936.51611730906,0,0,5062.7,38.7 +200000,1,40000,100,4989,0,73064.65640691575,76841.49191040442,80362.26728642671,0,0,4586.3,40.5 +200000,5,40000,100,3661,0,73313.49903227342,76641.96353006759,78346.77085396706,0,0,4241.5,42.0 diff --git a/scripts/verify-netem.sh b/scripts/verify-netem.sh new file mode 100755 index 0000000..5c452bc --- /dev/null +++ b/scripts/verify-netem.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# scripts/verify-netem.sh — confirm tc-netem is actually applying loss, in +# the direction you think it is. +# +# Usage: +# ./scripts/verify-netem.sh [interface] [loss_pct] +# +# peer-ip IP of the other machine (the simulator's IP when run on the CM5, +# or the CM5's IP when run on the Mac). +# interface Interface tc-netem is applied to. Default: eth0. +# loss_pct Loss percentage to apply. Default: 5. +# +# What it does: +# 1. Prints the current qdisc state (sanity check before). +# 2. Applies `netem loss %` on egress of . +# 3. Re-prints the qdisc state (confirms the rule is installed). +# 4. Sends 100 ICMP echo requests to and reports the observed loss. +# 5. Removes the qdisc on exit (trap), even on Ctrl-C. +# +# IMPORTANT direction caveat: +# tc qdisc rules apply to EGRESS only. If you run this on the CM5, it shapes +# CM5 → peer traffic. The peer → CM5 direction is unaffected. To drop traffic +# coming INTO this machine, you must either: +# (a) run netem on the SENDER side, or +# (b) attach an ifb (intermediate functional block) device for ingress. +# See the "Bidirectional loss" section in the script footer for the ifb recipe. + +set -euo pipefail + +PEER="${1:-}" +IFACE="${2:-eth0}" +LOSS="${3:-5}" + +if [[ -z "$PEER" ]]; then + echo "Usage: $0 [interface] [loss_pct]" + echo "Example: $0 192.168.1.42 eth0 5" + exit 1 +fi + +if [[ -t 1 ]]; then + BOLD=$'\033[1m'; DIM=$'\033[2m'; GREEN=$'\033[32m'; YELLOW=$'\033[33m' + RED=$'\033[31m'; RESET=$'\033[0m' +else + BOLD=; DIM=; GREEN=; YELLOW=; RED=; RESET= +fi +step() { printf '\n%s» %s%s\n' "$BOLD" "$1" "$RESET"; } +ok() { printf '%s ✓ %s%s\n' "$GREEN" "$1" "$RESET"; } +warn() { printf '%s ! %s%s\n' "$YELLOW" "$1" "$RESET"; } +fail() { printf '%s ✗ %s%s\n' "$RED" "$1" "$RESET"; } + +# Sanity: tc + ping + interface +for cmd in tc ping ip; do + command -v "$cmd" >/dev/null || { fail "missing: $cmd"; exit 1; } +done +ip link show "$IFACE" >/dev/null 2>&1 || { fail "interface $IFACE not found"; exit 1; } + +# Print the route to the peer so the user can see which iface the kernel +# actually uses — if it's not $IFACE, the netem rule won't apply. +step "Route to peer $PEER" +ROUTE_OUT="$(ip route get "$PEER" 2>&1 || true)" +printf ' %s\n' "$ROUTE_OUT" +ROUTE_IFACE="$(echo "$ROUTE_OUT" | awk '/dev/ {for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')" +if [[ -n "$ROUTE_IFACE" && "$ROUTE_IFACE" != "$IFACE" ]]; then + warn "kernel routes $PEER via '$ROUTE_IFACE' but you're shaping '$IFACE'." + warn "the netem rule will have NO effect on this peer's traffic." +fi + +# State BEFORE +step "Current qdisc on $IFACE (before)" +sudo tc qdisc show dev "$IFACE" | sed 's/^/ /' + +# Apply netem +step "Applying netem loss ${LOSS}% on $IFACE (egress)" +sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true +sudo tc qdisc add dev "$IFACE" root netem loss "${LOSS}%" +ok "qdisc installed" + +# Trap to clean up on any exit path +cleanup() { + step "Removing netem qdisc from $IFACE" + sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true + ok "qdisc removed; $IFACE back to default" +} +trap cleanup EXIT INT TERM + +# State AFTER install +step "Current qdisc on $IFACE (after netem applied)" +sudo tc qdisc show dev "$IFACE" | sed 's/^/ /' + +# Ping the peer and parse loss +step "Pinging $PEER with 100 echoes (egress goes through netem)" +PING_OUT="$(ping -c 100 -i 0.05 -W 1 "$PEER" 2>&1 || true)" +echo "$PING_OUT" | tail -3 | sed 's/^/ /' + +# Parse "X% packet loss" — works on both Linux and macOS ping output. +OBSERVED="$(echo "$PING_OUT" | grep -oE '[0-9.]+% packet loss' | head -1 | awk '{print $1}' | tr -d '%')" +if [[ -z "$OBSERVED" ]]; then + fail "could not parse ping output" + exit 1 +fi + +# Sanity bracket: configured loss is ±3 percentage points absolute is fine for n=100. +ABS_DELTA=$(awk -v o="$OBSERVED" -v l="$LOSS" 'BEGIN{d=o-l; if(d<0)d=-d; printf "%.1f", d}') +step "Result" +printf ' configured: %s%%\n observed: %s%%\n |delta|: %s pp\n' "$LOSS" "$OBSERVED" "$ABS_DELTA" + +if awk -v o="$OBSERVED" -v l="$LOSS" 'BEGIN{exit !(o > l*0.4 && o < l*2.0 + 3)}'; then + ok "loss is being applied in the egress direction of $IFACE" +else + fail "observed loss ($OBSERVED%) does not match configured ($LOSS%)" + warn "either the qdisc isn't routing as expected, or the kernel's netem" + warn "build doesn't include the loss module. Check 'modprobe sch_netem'." +fi + +# --------------------------------------------------------------------------- +# Bidirectional loss (info — not run by this script) +# --------------------------------------------------------------------------- +# tc qdisc shapes egress. To drop INCOMING traffic on the CM5 (e.g. T1 +# datagrams flowing from the Mac), redirect ingress to an ifb device and +# shape that: +# +# sudo modprobe ifb numifbs=1 +# sudo ip link set ifb0 up +# sudo tc qdisc add dev $IFACE handle ffff: ingress +# sudo tc filter add dev $IFACE parent ffff: protocol all u32 \ +# match u32 0 0 action mirred egress redirect dev ifb0 +# sudo tc qdisc add dev ifb0 root netem loss 5% +# +# Clean up: +# sudo tc qdisc del dev $IFACE ingress 2>/dev/null +# sudo tc qdisc del dev ifb0 root 2>/dev/null +# sudo ip link set ifb0 down +# +# Or — simpler — apply netem on BOTH ends with their respective egress qdiscs +# (run this script with the same loss_pct on the Mac too).