Results and script to verify netem

data/loopback/final_table.csv

@@ -1,13 +0,0 @@
-entities,loss_pct,devices,rate_hz,t1_received,t1_dropped,t1_p50_us,t1_p99_us,t1_p999_us,t2_p99_us,t3_rtt_us,hz,rss_mb
-10000,0,1428,100,5001,0,45238.33069752144,48411.85646346627,53098.29148381443,0,8630.865124336387,41257.9,12.0
-10000,1,1428,100,5001,0,45102.819074046034,47662.49174202053,50569.30401137826,0,42281.293387087724,22795.6,15.7
-10000,5,1428,100,5001,0,45030.712264117974,49449.14422462419,58098.785067702745,0,46383.544495318434,15843.7,19.0
-50000,0,7142,100,5001,0,44922.76813908747,48112.63150827307,55742.604812933576,0,9678.86600914163,12142.1,22.0
-50000,1,7142,100,5001,0,44797.160320888564,47757.912114388164,53204.59433455289,0,12706.894847406826,9835.6,26.4
-50000,5,7142,100,5001,0,44662.970225264624,47320.5542524692,50792.29917937587,0,45012.70358112902,8264.4,28.3
-100000,0,14285,100,5002,0,44538.08882376887,47681.5605522887,51477.49656030953,0,9963.662717402913,7115.0,30.1
-100000,1,14285,100,5001,0,44449.10166264818,47405.807955722245,52192.81032823135,0,14046.09958810414,6260.0,34.0
-100000,5,14285,100,5001,0,44369.16524374881,47510.21553980436,52213.691628413864,0,46023.16017305715,5547.8,40.9
-200000,0,28571,100,5001,0,44245.10534556269,47538.73022277268,50934.71690932847,0,9449.338570630583,5012.4,42.6
-200000,1,28571,100,5001,0,44121.392329231494,47159.93757056208,51539.306635015615,0,25644.900728832217,4565.4,44.2
-200000,5,28571,100,5001,0,44033.23772826498,47112.801204945215,50822.78370342773,0,45656.44371211316,4190.2,45.9

data/two_machine/final_table.csv

@@ -1,13 +1,13 @@
 entities,loss_pct,devices,rate_hz,t1_received,t1_dropped,t1_p50_us,t1_p99_us,t1_p999_us,t2_p99_us,t3_rtt_us,hz,rss_mb
-10000,0,2000,100,5002,0,88406.43436980792,92088.01036052403,95215.6914367099,0,0,41812.4,11.4
+10000,0,1428,100,5001,0,45238.33069752144,48411.85646346627,53098.29148381443,0,8630.865124336387,41257.9,12.0
-10000,1,2000,100,5001,0,88885.12040609193,92051.1825223156,94381.46931014946,0,0,23186.8,14.4
+10000,1,1428,100,5001,0,45102.819074046034,47662.49174202053,50569.30401137826,0,42281.293387087724,22795.6,15.7
-10000,5,2000,100,5002,0,70213.79159588156,74050.31194954121,79276.73877316424,0,0,16132.9,17.3
+10000,5,1428,100,5001,0,45030.712264117974,49449.14422462419,58098.785067702745,0,46383.544495318434,15843.7,19.0
-50000,0,10000,100,5001,0,70467.01677831604,74481.05169804857,78660.78554913378,0,0,12318.6,20.0
+50000,0,7142,100,5001,0,44922.76813908747,48112.63150827307,55742.604812933576,0,9678.86600914163,12142.1,22.0
-50000,1,10000,100,4999,0,70990.40685911797,74854.38952456272,80926.7766369622,0,0,9959.4,23.7
+50000,1,7142,100,5001,0,44797.160320888564,47757.912114388164,53204.59433455289,0,12706.894847406826,9835.6,26.4
-50000,5,10000,100,5001,0,71389.06834919532,74944.26870901692,79070.8869749687,0,0,8396.9,25.4
+50000,5,7142,100,5001,0,44662.970225264624,47320.5542524692,50792.29917937587,0,45012.70358112902,8264.4,28.3
-100000,0,20000,100,5000,0,71675.19649834004,75365.13393797085,78976.05881855593,0,0,7224.8,27.1
+100000,0,14285,100,5002,0,44538.08882376887,47681.5605522887,51477.49656030953,0,9963.662717402913,7115.0,30.1
-100000,1,20000,100,4998,0,72106.54041649138,76718.64382761203,81527.85603693608,0,0,6353.7,30.9
+100000,1,14285,100,5001,0,44449.10166264818,47405.807955722245,52192.81032823135,0,14046.09958810414,6260.0,34.0
-100000,5,20000,100,4997,0,72453.48380965949,75894.54063901275,78049.6180715917,0,0,5660.0,36.9
+100000,5,14285,100,5001,0,44369.16524374881,47510.21553980436,52213.691628413864,0,46023.16017305715,5547.8,40.9
-200000,0,40000,100,4992,0,72758.4283780017,77815.82008892013,81936.51611730906,0,0,5062.7,38.7
+200000,0,28571,100,5001,0,44245.10534556269,47538.73022277268,50934.71690932847,0,9449.338570630583,5012.4,42.6
-200000,1,40000,100,4989,0,73064.65640691575,76841.49191040442,80362.26728642671,0,0,4586.3,40.5
+200000,1,28571,100,5001,0,44121.392329231494,47159.93757056208,51539.306635015615,0,25644.900728832217,4565.4,44.2
-200000,5,40000,100,3661,0,73313.49903227342,76641.96353006759,78346.77085396706,0,0,4241.5,42.0
+200000,5,28571,100,5001,0,44033.23772826498,47112.801204945215,50822.78370342773,0,45656.44371211316,4190.2,45.9

data/two_machine_OLD/final_table.csv

@@ -0,0 +1,13 @@
+entities,loss_pct,devices,rate_hz,t1_received,t1_dropped,t1_p50_us,t1_p99_us,t1_p999_us,t2_p99_us,t3_rtt_us,hz,rss_mb
+10000,0,2000,100,5002,0,88406.43436980792,92088.01036052403,95215.6914367099,0,0,41812.4,11.4
+10000,1,2000,100,5001,0,88885.12040609193,92051.1825223156,94381.46931014946,0,0,23186.8,14.4
+10000,5,2000,100,5002,0,70213.79159588156,74050.31194954121,79276.73877316424,0,0,16132.9,17.3
+50000,0,10000,100,5001,0,70467.01677831604,74481.05169804857,78660.78554913378,0,0,12318.6,20.0
+50000,1,10000,100,4999,0,70990.40685911797,74854.38952456272,80926.7766369622,0,0,9959.4,23.7
+50000,5,10000,100,5001,0,71389.06834919532,74944.26870901692,79070.8869749687,0,0,8396.9,25.4
+100000,0,20000,100,5000,0,71675.19649834004,75365.13393797085,78976.05881855593,0,0,7224.8,27.1
+100000,1,20000,100,4998,0,72106.54041649138,76718.64382761203,81527.85603693608,0,0,6353.7,30.9
+100000,5,20000,100,4997,0,72453.48380965949,75894.54063901275,78049.6180715917,0,0,5660.0,36.9
+200000,0,40000,100,4992,0,72758.4283780017,77815.82008892013,81936.51611730906,0,0,5062.7,38.7
+200000,1,40000,100,4989,0,73064.65640691575,76841.49191040442,80362.26728642671,0,0,4586.3,40.5
+200000,5,40000,100,3661,0,73313.49903227342,76641.96353006759,78346.77085396706,0,0,4241.5,42.0

scripts/verify-netem.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# scripts/verify-netem.sh — confirm tc-netem is actually applying loss, in
+# the direction you think it is.
+#
+# Usage:
+#   ./scripts/verify-netem.sh <peer-ip> [interface] [loss_pct]
+#
+#   peer-ip     IP of the other machine (the simulator's IP when run on the CM5,
+#               or the CM5's IP when run on the Mac).
+#   interface   Interface tc-netem is applied to. Default: eth0.
+#   loss_pct    Loss percentage to apply. Default: 5.
+#
+# What it does:
+#   1. Prints the current qdisc state (sanity check before).
+#   2. Applies `netem loss <loss_pct>%` on egress of <interface>.
+#   3. Re-prints the qdisc state (confirms the rule is installed).
+#   4. Sends 100 ICMP echo requests to <peer-ip> and reports the observed loss.
+#   5. Removes the qdisc on exit (trap), even on Ctrl-C.
+#
+# IMPORTANT direction caveat:
+#   tc qdisc rules apply to EGRESS only. If you run this on the CM5, it shapes
+#   CM5 → peer traffic. The peer → CM5 direction is unaffected. To drop traffic
+#   coming INTO this machine, you must either:
+#     (a) run netem on the SENDER side, or
+#     (b) attach an ifb (intermediate functional block) device for ingress.
+#   See the "Bidirectional loss" section in the script footer for the ifb recipe.
+
+set -euo pipefail
+
+PEER="${1:-}"
+IFACE="${2:-eth0}"
+LOSS="${3:-5}"
+
+if [[ -z "$PEER" ]]; then
+    echo "Usage: $0 <peer-ip> [interface] [loss_pct]"
+    echo "Example: $0 192.168.1.42 eth0 5"
+    exit 1
+fi
+
+if [[ -t 1 ]]; then
+    BOLD=$'\033[1m'; DIM=$'\033[2m'; GREEN=$'\033[32m'; YELLOW=$'\033[33m'
+    RED=$'\033[31m'; RESET=$'\033[0m'
+else
+    BOLD=; DIM=; GREEN=; YELLOW=; RED=; RESET=
+fi
+step()  { printf '\n%s» %s%s\n' "$BOLD" "$1" "$RESET"; }
+ok()    { printf '%s  ✓ %s%s\n' "$GREEN" "$1" "$RESET"; }
+warn()  { printf '%s  ! %s%s\n' "$YELLOW" "$1" "$RESET"; }
+fail()  { printf '%s  ✗ %s%s\n' "$RED" "$1" "$RESET"; }
+
+# Sanity: tc + ping + interface
+for cmd in tc ping ip; do
+    command -v "$cmd" >/dev/null || { fail "missing: $cmd"; exit 1; }
+done
+ip link show "$IFACE" >/dev/null 2>&1 || { fail "interface $IFACE not found"; exit 1; }
+
+# Print the route to the peer so the user can see which iface the kernel
+# actually uses — if it's not $IFACE, the netem rule won't apply.
+step "Route to peer $PEER"
+ROUTE_OUT="$(ip route get "$PEER" 2>&1 || true)"
+printf '  %s\n' "$ROUTE_OUT"
+ROUTE_IFACE="$(echo "$ROUTE_OUT" | awk '/dev/ {for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')"
+if [[ -n "$ROUTE_IFACE" && "$ROUTE_IFACE" != "$IFACE" ]]; then
+    warn "kernel routes $PEER via '$ROUTE_IFACE' but you're shaping '$IFACE'."
+    warn "the netem rule will have NO effect on this peer's traffic."
+fi
+
+# State BEFORE
+step "Current qdisc on $IFACE (before)"
+sudo tc qdisc show dev "$IFACE" | sed 's/^/  /'
+
+# Apply netem
+step "Applying netem loss ${LOSS}% on $IFACE (egress)"
+sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
+sudo tc qdisc add dev "$IFACE" root netem loss "${LOSS}%"
+ok "qdisc installed"
+
+# Trap to clean up on any exit path
+cleanup() {
+    step "Removing netem qdisc from $IFACE"
+    sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
+    ok "qdisc removed; $IFACE back to default"
+}
+trap cleanup EXIT INT TERM
+
+# State AFTER install
+step "Current qdisc on $IFACE (after netem applied)"
+sudo tc qdisc show dev "$IFACE" | sed 's/^/  /'
+
+# Ping the peer and parse loss
+step "Pinging $PEER with 100 echoes (egress goes through netem)"
+PING_OUT="$(ping -c 100 -i 0.05 -W 1 "$PEER" 2>&1 || true)"
+echo "$PING_OUT" | tail -3 | sed 's/^/  /'
+
+# Parse "X% packet loss" — works on both Linux and macOS ping output.
+OBSERVED="$(echo "$PING_OUT" | grep -oE '[0-9.]+% packet loss' | head -1 | awk '{print $1}' | tr -d '%')"
+if [[ -z "$OBSERVED" ]]; then
+    fail "could not parse ping output"
+    exit 1
+fi
+
+# Sanity bracket: configured loss is ±3 percentage points absolute is fine for n=100.
+ABS_DELTA=$(awk -v o="$OBSERVED" -v l="$LOSS" 'BEGIN{d=o-l; if(d<0)d=-d; printf "%.1f", d}')
+step "Result"
+printf '  configured: %s%%\n  observed:   %s%%\n  |delta|:    %s pp\n' "$LOSS" "$OBSERVED" "$ABS_DELTA"
+
+if awk -v o="$OBSERVED" -v l="$LOSS" 'BEGIN{exit !(o > l*0.4 && o < l*2.0 + 3)}'; then
+    ok "loss is being applied in the egress direction of $IFACE"
+else
+    fail "observed loss ($OBSERVED%) does not match configured ($LOSS%)"
+    warn "either the qdisc isn't routing as expected, or the kernel's netem"
+    warn "build doesn't include the loss module. Check 'modprobe sch_netem'."
+fi
+
+# ---------------------------------------------------------------------------
+# Bidirectional loss (info — not run by this script)
+# ---------------------------------------------------------------------------
+# tc qdisc shapes egress. To drop INCOMING traffic on the CM5 (e.g. T1
+# datagrams flowing from the Mac), redirect ingress to an ifb device and
+# shape that:
+#
+#   sudo modprobe ifb numifbs=1
+#   sudo ip link set ifb0 up
+#   sudo tc qdisc add dev $IFACE handle ffff: ingress
+#   sudo tc filter add dev $IFACE parent ffff: protocol all u32 \
+#       match u32 0 0 action mirred egress redirect dev ifb0
+#   sudo tc qdisc add dev ifb0 root netem loss 5%
+#
+# Clean up:
+#   sudo tc qdisc del dev $IFACE ingress 2>/dev/null
+#   sudo tc qdisc del dev ifb0 root 2>/dev/null
+#   sudo ip link set ifb0 down
+#
+# Or — simpler — apply netem on BOTH ends with their respective egress qdiscs
+# (run this script with the same loss_pct on the Mac too).