diff --git a/scripts/bench-loss.sh b/scripts/bench-loss.sh index 5ceeabd..1a09238 100755 --- a/scripts/bench-loss.sh +++ b/scripts/bench-loss.sh @@ -16,6 +16,10 @@ RATE_HZ="${RATE_HZ:-100}" BUILD="${BUILD:-release}" IFACE="${IFACE:-eth0}" RUN_SIMULATOR="${RUN_SIMULATOR:-1}" +# Bidirectional loss via ifb (ingress redirect). Set BIDI=0 to disable and fall +# back to egress-only shaping on $IFACE. +BIDI="${BIDI:-1}" +IFB_DEV="${IFB_DEV:-ifb0}" OUT_CSV="${OUT_CSV:-data/loopback/final_table.csv}" @@ -45,6 +49,67 @@ for port in 9000 9100; do done [[ -f certs/server.crt ]] || make certs >/dev/null +# --- netem helpers ----------------------------------------------------------- +# tc qdiscs are egress-only. To shape ingress (traffic arriving on $IFACE) we +# install an ingress qdisc that redirects every packet to an ifb device, then +# put the netem qdisc on the ifb's egress. Net effect: both directions of +# $IFACE see the configured loss percentage. +# +# Egress-only fallback (BIDI=0) keeps the historical behaviour: shape outgoing +# CM5 → peer traffic only. + +netem_init() { + [[ "$HAS_TC" -eq 1 && "$BIDI" -eq 1 ]] || return 0 + # Load the ifb module (idempotent on modern kernels; ignored if built in). + sudo modprobe ifb numifbs=1 2>/dev/null || true + if ! ip link show "$IFB_DEV" >/dev/null 2>&1; then + fail "ifb device $IFB_DEV not present after modprobe; BIDI mode unavailable" + echo " - check 'modprobe ifb' on this kernel, or run with BIDI=0" + return 1 + fi + sudo ip link set "$IFB_DEV" up +} + +netem_clear() { + [[ "$HAS_TC" -eq 1 ]] || return 0 + sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true + sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true + if [[ "$BIDI" -eq 1 ]]; then + sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true + fi +} + +# Apply $1% loss in both directions (or egress-only when BIDI=0). +netem_apply() { + local pct="$1" + [[ "$HAS_TC" -eq 1 ]] || return 0 + netem_clear + [[ "$pct" -gt 0 ]] || return 0 + # Egress: outgoing from $IFACE. + sudo tc qdisc add dev "$IFACE" root netem loss "${pct}%" 2>/dev/null || { + echo "Warning: failed to apply egress netem loss on $IFACE." + return 1 + } + if [[ "$BIDI" -eq 1 ]]; then + # Ingress: incoming on $IFACE, redirected to $IFB_DEV egress and shaped there. + sudo tc qdisc add dev "$IFACE" handle ffff: ingress 2>/dev/null || { + echo "Warning: failed to add ingress qdisc on $IFACE." + return 1 + } + sudo tc filter add dev "$IFACE" parent ffff: protocol all u32 \ + match u32 0 0 action mirred egress redirect dev "$IFB_DEV" 2>/dev/null || { + echo "Warning: failed to install ingress redirect filter." + return 1 + } + sudo tc qdisc add dev "$IFB_DEV" root netem loss "${pct}%" 2>/dev/null || { + echo "Warning: failed to apply netem on $IFB_DEV." + return 1 + } + fi +} + +netem_init || true + step "Building ($BUILD)" if [[ "$BUILD" == "release" ]]; then if [[ "$RUN_SIMULATOR" -eq 1 ]]; then @@ -84,9 +149,7 @@ done cleanup() { [[ -n "${SIM_PID:-}" ]] && kill -TERM "$SIM_PID" 2>/dev/null || true [[ -n "${SUBSTRATE_PID:-}" ]] && kill -TERM "$SUBSTRATE_PID" 2>/dev/null || true - if [[ "$HAS_TC" -eq 1 ]]; then - sudo tc qdisc del dev $IFACE root 2>/dev/null || true - fi + netem_clear wait 2>/dev/null || true } trap cleanup EXIT INT TERM @@ -115,15 +178,8 @@ for entities in "${ENTITIES_LIST[@]}"; do devices=$(( entities / 7 )) for loss in "${LOSS_LIST[@]}"; do - # Apply tc netem loss - if [[ "$HAS_TC" -eq 1 ]]; then - sudo tc qdisc del dev $IFACE root 2>/dev/null || true - if [[ "$loss" -gt 0 ]]; then - sudo tc qdisc add dev $IFACE root netem loss ${loss}% 2>/dev/null || { - echo "Warning: failed to apply tc netem loss on interface $IFACE." - } - fi - fi + # Apply tc-netem loss in both directions (or egress-only when BIDI=0). + netem_apply "$loss" if [[ "$RUN_SIMULATOR" -eq 1 ]]; then sim_args=( @@ -178,8 +234,6 @@ for entities in "${ENTITIES_LIST[@]}"; do done done -if [[ "$HAS_TC" -eq 1 ]]; then - sudo tc qdisc del dev $IFACE root 2>/dev/null || true -fi +netem_clear printf '\n%sCSV written to:%s %s\n' "$DIM" "$RESET" "$OUT_CSV" diff --git a/scripts/verify-netem.sh b/scripts/verify-netem.sh index 5c452bc..71ce2a4 100755 --- a/scripts/verify-netem.sh +++ b/scripts/verify-netem.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # scripts/verify-netem.sh — confirm tc-netem is actually applying loss, in -# the direction you think it is. +# the direction(s) you think it is. # # Usage: # ./scripts/verify-netem.sh [interface] [loss_pct] @@ -10,26 +10,32 @@ # interface Interface tc-netem is applied to. Default: eth0. # loss_pct Loss percentage to apply. Default: 5. # +# Modes (env var): +# BIDI=0 (default) Egress-only. Shapes outgoing traffic from . +# Use a single ping from this host to verify. +# BIDI=1 Bidirectional via ifb ingress redirect. Shapes BOTH +# outgoing AND incoming traffic on . Pings +# run from THIS host verify egress; the script also +# prompts you to ping back FROM THE PEER to verify +# ingress (the script holds the qdisc up until you press +# Enter, then tears everything down). +# # What it does: # 1. Prints the current qdisc state (sanity check before). -# 2. Applies `netem loss %` on egress of . +# 2. Applies the configured netem loss (egress, or egress + ingress). # 3. Re-prints the qdisc state (confirms the rule is installed). # 4. Sends 100 ICMP echo requests to and reports the observed loss. -# 5. Removes the qdisc on exit (trap), even on Ctrl-C. -# -# IMPORTANT direction caveat: -# tc qdisc rules apply to EGRESS only. If you run this on the CM5, it shapes -# CM5 → peer traffic. The peer → CM5 direction is unaffected. To drop traffic -# coming INTO this machine, you must either: -# (a) run netem on the SENDER side, or -# (b) attach an ifb (intermediate functional block) device for ingress. -# See the "Bidirectional loss" section in the script footer for the ifb recipe. +# 5. BIDI=1 only: waits for you to run `ping -c 100 ` from the +# peer machine and report what it saw. +# 6. Removes the qdiscs (and brings ifb0 down) on exit, even on Ctrl-C. set -euo pipefail PEER="${1:-}" IFACE="${2:-eth0}" LOSS="${3:-5}" +BIDI="${BIDI:-0}" +IFB_DEV="${IFB_DEV:-ifb0}" if [[ -z "$PEER" ]]; then echo "Usage: $0 [interface] [loss_pct]" @@ -69,23 +75,57 @@ fi step "Current qdisc on $IFACE (before)" sudo tc qdisc show dev "$IFACE" | sed 's/^/ /' +# If bidi, ensure ifb device is up before installing qdiscs. +if [[ "$BIDI" -eq 1 ]]; then + step "Bringing up $IFB_DEV (BIDI mode)" + sudo modprobe ifb numifbs=1 2>/dev/null || true + if ! ip link show "$IFB_DEV" >/dev/null 2>&1; then + fail "ifb device $IFB_DEV not present after modprobe; cannot run BIDI mode" + exit 1 + fi + sudo ip link set "$IFB_DEV" up + ok "$IFB_DEV is up" +fi + # Apply netem -step "Applying netem loss ${LOSS}% on $IFACE (egress)" +if [[ "$BIDI" -eq 1 ]]; then + step "Applying netem loss ${LOSS}% on $IFACE (egress + ingress via $IFB_DEV)" +else + step "Applying netem loss ${LOSS}% on $IFACE (egress only)" +fi sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true +sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true +[[ "$BIDI" -eq 1 ]] && sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true sudo tc qdisc add dev "$IFACE" root netem loss "${LOSS}%" -ok "qdisc installed" +if [[ "$BIDI" -eq 1 ]]; then + sudo tc qdisc add dev "$IFACE" handle ffff: ingress + sudo tc filter add dev "$IFACE" parent ffff: protocol all u32 \ + match u32 0 0 action mirred egress redirect dev "$IFB_DEV" + sudo tc qdisc add dev "$IFB_DEV" root netem loss "${LOSS}%" +fi +ok "qdisc(s) installed" # Trap to clean up on any exit path cleanup() { - step "Removing netem qdisc from $IFACE" + step "Removing netem qdiscs" sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true - ok "qdisc removed; $IFACE back to default" + sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true + if [[ "$BIDI" -eq 1 ]]; then + sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true + sudo ip link set "$IFB_DEV" down 2>/dev/null || true + fi + ok "qdiscs removed; $IFACE back to default" } trap cleanup EXIT INT TERM # State AFTER install -step "Current qdisc on $IFACE (after netem applied)" +step "Current qdisc state (after netem applied)" sudo tc qdisc show dev "$IFACE" | sed 's/^/ /' +if [[ "$BIDI" -eq 1 ]]; then + sudo tc qdisc show dev "$IFB_DEV" | sed 's/^/ /' + echo " filter on $IFACE ingress:" + sudo tc filter show dev "$IFACE" parent ffff: 2>/dev/null | sed 's/^/ /' +fi # Ping the peer and parse loss step "Pinging $PEER with 100 echoes (egress goes through netem)" @@ -112,24 +152,17 @@ else warn "build doesn't include the loss module. Check 'modprobe sch_netem'." fi -# --------------------------------------------------------------------------- -# Bidirectional loss (info — not run by this script) -# --------------------------------------------------------------------------- -# tc qdisc shapes egress. To drop INCOMING traffic on the CM5 (e.g. T1 -# datagrams flowing from the Mac), redirect ingress to an ifb device and -# shape that: -# -# sudo modprobe ifb numifbs=1 -# sudo ip link set ifb0 up -# sudo tc qdisc add dev $IFACE handle ffff: ingress -# sudo tc filter add dev $IFACE parent ffff: protocol all u32 \ -# match u32 0 0 action mirred egress redirect dev ifb0 -# sudo tc qdisc add dev ifb0 root netem loss 5% -# -# Clean up: -# sudo tc qdisc del dev $IFACE ingress 2>/dev/null -# sudo tc qdisc del dev ifb0 root 2>/dev/null -# sudo ip link set ifb0 down -# -# Or — simpler — apply netem on BOTH ends with their respective egress qdiscs -# (run this script with the same loss_pct on the Mac too). +if [[ "$BIDI" -eq 1 ]]; then + step "Now verify the INGRESS direction" + THIS_IP="$(ip -4 addr show dev "$IFACE" | awk '/inet / {print $2}' | cut -d/ -f1 | head -1)" + cat <} + You should see ~${LOSS}% packet loss in the peer's ping output. That confirms + ingress shaping is dropping packets arriving here on $IFACE. + + Press Enter when done to tear everything down. +EOF + read -r _