Update to scripts to better handle netem
This commit is contained in:
Valère Plantevin
@@ -16,6 +16,10 @@ RATE_HZ="${RATE_HZ:-100}"
|
||||
BUILD="${BUILD:-release}"
|
||||
IFACE="${IFACE:-eth0}"
|
||||
RUN_SIMULATOR="${RUN_SIMULATOR:-1}"
|
||||
# Bidirectional loss via ifb (ingress redirect). Set BIDI=0 to disable and fall
|
||||
# back to egress-only shaping on $IFACE.
|
||||
BIDI="${BIDI:-1}"
|
||||
IFB_DEV="${IFB_DEV:-ifb0}"
|
||||
|
||||
OUT_CSV="${OUT_CSV:-data/loopback/final_table.csv}"
|
||||
|
||||
@@ -45,6 +49,67 @@ for port in 9000 9100; do
|
||||
done
|
||||
[[ -f certs/server.crt ]] || make certs >/dev/null
|
||||
|
||||
# --- netem helpers -----------------------------------------------------------
|
||||
# tc qdiscs are egress-only. To shape ingress (traffic arriving on $IFACE) we
|
||||
# install an ingress qdisc that redirects every packet to an ifb device, then
|
||||
# put the netem qdisc on the ifb's egress. Net effect: both directions of
|
||||
# $IFACE see the configured loss percentage.
|
||||
#
|
||||
# Egress-only fallback (BIDI=0) keeps the historical behaviour: shape outgoing
|
||||
# CM5 → peer traffic only.
|
||||
|
||||
netem_init() {
|
||||
[[ "$HAS_TC" -eq 1 && "$BIDI" -eq 1 ]] || return 0
|
||||
# Load the ifb module (idempotent on modern kernels; ignored if built in).
|
||||
sudo modprobe ifb numifbs=1 2>/dev/null || true
|
||||
if ! ip link show "$IFB_DEV" >/dev/null 2>&1; then
|
||||
fail "ifb device $IFB_DEV not present after modprobe; BIDI mode unavailable"
|
||||
echo " - check 'modprobe ifb' on this kernel, or run with BIDI=0"
|
||||
return 1
|
||||
fi
|
||||
sudo ip link set "$IFB_DEV" up
|
||||
}
|
||||
|
||||
netem_clear() {
|
||||
[[ "$HAS_TC" -eq 1 ]] || return 0
|
||||
sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
|
||||
sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Apply $1% loss in both directions (or egress-only when BIDI=0).
|
||||
netem_apply() {
|
||||
local pct="$1"
|
||||
[[ "$HAS_TC" -eq 1 ]] || return 0
|
||||
netem_clear
|
||||
[[ "$pct" -gt 0 ]] || return 0
|
||||
# Egress: outgoing from $IFACE.
|
||||
sudo tc qdisc add dev "$IFACE" root netem loss "${pct}%" 2>/dev/null || {
|
||||
echo "Warning: failed to apply egress netem loss on $IFACE."
|
||||
return 1
|
||||
}
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
# Ingress: incoming on $IFACE, redirected to $IFB_DEV egress and shaped there.
|
||||
sudo tc qdisc add dev "$IFACE" handle ffff: ingress 2>/dev/null || {
|
||||
echo "Warning: failed to add ingress qdisc on $IFACE."
|
||||
return 1
|
||||
}
|
||||
sudo tc filter add dev "$IFACE" parent ffff: protocol all u32 \
|
||||
match u32 0 0 action mirred egress redirect dev "$IFB_DEV" 2>/dev/null || {
|
||||
echo "Warning: failed to install ingress redirect filter."
|
||||
return 1
|
||||
}
|
||||
sudo tc qdisc add dev "$IFB_DEV" root netem loss "${pct}%" 2>/dev/null || {
|
||||
echo "Warning: failed to apply netem on $IFB_DEV."
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
}
|
||||
|
||||
netem_init || true
|
||||
|
||||
step "Building ($BUILD)"
|
||||
if [[ "$BUILD" == "release" ]]; then
|
||||
if [[ "$RUN_SIMULATOR" -eq 1 ]]; then
|
||||
@@ -84,9 +149,7 @@ done
|
||||
cleanup() {
|
||||
[[ -n "${SIM_PID:-}" ]] && kill -TERM "$SIM_PID" 2>/dev/null || true
|
||||
[[ -n "${SUBSTRATE_PID:-}" ]] && kill -TERM "$SUBSTRATE_PID" 2>/dev/null || true
|
||||
if [[ "$HAS_TC" -eq 1 ]]; then
|
||||
sudo tc qdisc del dev $IFACE root 2>/dev/null || true
|
||||
fi
|
||||
netem_clear
|
||||
wait 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
@@ -115,15 +178,8 @@ for entities in "${ENTITIES_LIST[@]}"; do
|
||||
devices=$(( entities / 7 ))
|
||||
|
||||
for loss in "${LOSS_LIST[@]}"; do
|
||||
# Apply tc netem loss
|
||||
if [[ "$HAS_TC" -eq 1 ]]; then
|
||||
sudo tc qdisc del dev $IFACE root 2>/dev/null || true
|
||||
if [[ "$loss" -gt 0 ]]; then
|
||||
sudo tc qdisc add dev $IFACE root netem loss ${loss}% 2>/dev/null || {
|
||||
echo "Warning: failed to apply tc netem loss on interface $IFACE."
|
||||
}
|
||||
fi
|
||||
fi
|
||||
# Apply tc-netem loss in both directions (or egress-only when BIDI=0).
|
||||
netem_apply "$loss"
|
||||
|
||||
if [[ "$RUN_SIMULATOR" -eq 1 ]]; then
|
||||
sim_args=(
|
||||
@@ -178,8 +234,6 @@ for entities in "${ENTITIES_LIST[@]}"; do
|
||||
done
|
||||
done
|
||||
|
||||
if [[ "$HAS_TC" -eq 1 ]]; then
|
||||
sudo tc qdisc del dev $IFACE root 2>/dev/null || true
|
||||
fi
|
||||
netem_clear
|
||||
|
||||
printf '\n%sCSV written to:%s %s\n' "$DIM" "$RESET" "$OUT_CSV"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/verify-netem.sh — confirm tc-netem is actually applying loss, in
|
||||
# the direction you think it is.
|
||||
# the direction(s) you think it is.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/verify-netem.sh <peer-ip> [interface] [loss_pct]
|
||||
@@ -10,26 +10,32 @@
|
||||
# interface Interface tc-netem is applied to. Default: eth0.
|
||||
# loss_pct Loss percentage to apply. Default: 5.
|
||||
#
|
||||
# Modes (env var):
|
||||
# BIDI=0 (default) Egress-only. Shapes outgoing traffic from <interface>.
|
||||
# Use a single ping from this host to verify.
|
||||
# BIDI=1 Bidirectional via ifb ingress redirect. Shapes BOTH
|
||||
# outgoing AND incoming traffic on <interface>. Pings
|
||||
# run from THIS host verify egress; the script also
|
||||
# prompts you to ping back FROM THE PEER to verify
|
||||
# ingress (the script holds the qdisc up until you press
|
||||
# Enter, then tears everything down).
|
||||
#
|
||||
# What it does:
|
||||
# 1. Prints the current qdisc state (sanity check before).
|
||||
# 2. Applies `netem loss <loss_pct>%` on egress of <interface>.
|
||||
# 2. Applies the configured netem loss (egress, or egress + ingress).
|
||||
# 3. Re-prints the qdisc state (confirms the rule is installed).
|
||||
# 4. Sends 100 ICMP echo requests to <peer-ip> and reports the observed loss.
|
||||
# 5. Removes the qdisc on exit (trap), even on Ctrl-C.
|
||||
#
|
||||
# IMPORTANT direction caveat:
|
||||
# tc qdisc rules apply to EGRESS only. If you run this on the CM5, it shapes
|
||||
# CM5 → peer traffic. The peer → CM5 direction is unaffected. To drop traffic
|
||||
# coming INTO this machine, you must either:
|
||||
# (a) run netem on the SENDER side, or
|
||||
# (b) attach an ifb (intermediate functional block) device for ingress.
|
||||
# See the "Bidirectional loss" section in the script footer for the ifb recipe.
|
||||
# 5. BIDI=1 only: waits for you to run `ping -c 100 <this-host>` from the
|
||||
# peer machine and report what it saw.
|
||||
# 6. Removes the qdiscs (and brings ifb0 down) on exit, even on Ctrl-C.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PEER="${1:-}"
|
||||
IFACE="${2:-eth0}"
|
||||
LOSS="${3:-5}"
|
||||
BIDI="${BIDI:-0}"
|
||||
IFB_DEV="${IFB_DEV:-ifb0}"
|
||||
|
||||
if [[ -z "$PEER" ]]; then
|
||||
echo "Usage: $0 <peer-ip> [interface] [loss_pct]"
|
||||
@@ -69,23 +75,57 @@ fi
|
||||
step "Current qdisc on $IFACE (before)"
|
||||
sudo tc qdisc show dev "$IFACE" | sed 's/^/ /'
|
||||
|
||||
# If bidi, ensure ifb device is up before installing qdiscs.
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
step "Bringing up $IFB_DEV (BIDI mode)"
|
||||
sudo modprobe ifb numifbs=1 2>/dev/null || true
|
||||
if ! ip link show "$IFB_DEV" >/dev/null 2>&1; then
|
||||
fail "ifb device $IFB_DEV not present after modprobe; cannot run BIDI mode"
|
||||
exit 1
|
||||
fi
|
||||
sudo ip link set "$IFB_DEV" up
|
||||
ok "$IFB_DEV is up"
|
||||
fi
|
||||
|
||||
# Apply netem
|
||||
step "Applying netem loss ${LOSS}% on $IFACE (egress)"
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
step "Applying netem loss ${LOSS}% on $IFACE (egress + ingress via $IFB_DEV)"
|
||||
else
|
||||
step "Applying netem loss ${LOSS}% on $IFACE (egress only)"
|
||||
fi
|
||||
sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
|
||||
sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true
|
||||
[[ "$BIDI" -eq 1 ]] && sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true
|
||||
sudo tc qdisc add dev "$IFACE" root netem loss "${LOSS}%"
|
||||
ok "qdisc installed"
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
sudo tc qdisc add dev "$IFACE" handle ffff: ingress
|
||||
sudo tc filter add dev "$IFACE" parent ffff: protocol all u32 \
|
||||
match u32 0 0 action mirred egress redirect dev "$IFB_DEV"
|
||||
sudo tc qdisc add dev "$IFB_DEV" root netem loss "${LOSS}%"
|
||||
fi
|
||||
ok "qdisc(s) installed"
|
||||
|
||||
# Trap to clean up on any exit path
|
||||
cleanup() {
|
||||
step "Removing netem qdisc from $IFACE"
|
||||
step "Removing netem qdiscs"
|
||||
sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
|
||||
ok "qdisc removed; $IFACE back to default"
|
||||
sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true
|
||||
sudo ip link set "$IFB_DEV" down 2>/dev/null || true
|
||||
fi
|
||||
ok "qdiscs removed; $IFACE back to default"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# State AFTER install
|
||||
step "Current qdisc on $IFACE (after netem applied)"
|
||||
step "Current qdisc state (after netem applied)"
|
||||
sudo tc qdisc show dev "$IFACE" | sed 's/^/ /'
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
sudo tc qdisc show dev "$IFB_DEV" | sed 's/^/ /'
|
||||
echo " filter on $IFACE ingress:"
|
||||
sudo tc filter show dev "$IFACE" parent ffff: 2>/dev/null | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# Ping the peer and parse loss
|
||||
step "Pinging $PEER with 100 echoes (egress goes through netem)"
|
||||
@@ -112,24 +152,17 @@ else
|
||||
warn "build doesn't include the loss module. Check 'modprobe sch_netem'."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bidirectional loss (info — not run by this script)
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc qdisc shapes egress. To drop INCOMING traffic on the CM5 (e.g. T1
|
||||
# datagrams flowing from the Mac), redirect ingress to an ifb device and
|
||||
# shape that:
|
||||
#
|
||||
# sudo modprobe ifb numifbs=1
|
||||
# sudo ip link set ifb0 up
|
||||
# sudo tc qdisc add dev $IFACE handle ffff: ingress
|
||||
# sudo tc filter add dev $IFACE parent ffff: protocol all u32 \
|
||||
# match u32 0 0 action mirred egress redirect dev ifb0
|
||||
# sudo tc qdisc add dev ifb0 root netem loss 5%
|
||||
#
|
||||
# Clean up:
|
||||
# sudo tc qdisc del dev $IFACE ingress 2>/dev/null
|
||||
# sudo tc qdisc del dev ifb0 root 2>/dev/null
|
||||
# sudo ip link set ifb0 down
|
||||
#
|
||||
# Or — simpler — apply netem on BOTH ends with their respective egress qdiscs
|
||||
# (run this script with the same loss_pct on the Mac too).
|
||||
if [[ "$BIDI" -eq 1 ]]; then
|
||||
step "Now verify the INGRESS direction"
|
||||
THIS_IP="$(ip -4 addr show dev "$IFACE" | awk '/inet / {print $2}' | cut -d/ -f1 | head -1)"
|
||||
cat <<EOF
|
||||
From the peer machine, run:
|
||||
ping -c 100 -i 0.05 ${THIS_IP:-<this host\'s IP on $IFACE>}
|
||||
You should see ~${LOSS}% packet loss in the peer's ping output. That confirms
|
||||
ingress shaping is dropping packets arriving here on $IFACE.
|
||||
|
||||
Press Enter when done to tear everything down.
|
||||
EOF
|
||||
read -r _ </dev/tty || true
|
||||
fi
|
||||
|
||||
|
||||
Reference in New Issue
Block a user