Update to scripts to better handle netem

scripts/bench-loss.sh

@@ -16,6 +16,10 @@ RATE_HZ="${RATE_HZ:-100}"
 BUILD="${BUILD:-release}"
 IFACE="${IFACE:-eth0}"
 RUN_SIMULATOR="${RUN_SIMULATOR:-1}"
+# Bidirectional loss via ifb (ingress redirect). Set BIDI=0 to disable and fall
+# back to egress-only shaping on $IFACE.
+BIDI="${BIDI:-1}"
+IFB_DEV="${IFB_DEV:-ifb0}"
 
 OUT_CSV="${OUT_CSV:-data/loopback/final_table.csv}"
 
@@ -45,6 +49,67 @@ for port in 9000 9100; do
 done
 [[ -f certs/server.crt ]] || make certs >/dev/null
 
+# --- netem helpers -----------------------------------------------------------
+# tc qdiscs are egress-only. To shape ingress (traffic arriving on $IFACE) we
+# install an ingress qdisc that redirects every packet to an ifb device, then
+# put the netem qdisc on the ifb's egress. Net effect: both directions of
+# $IFACE see the configured loss percentage.
+#
+# Egress-only fallback (BIDI=0) keeps the historical behaviour: shape outgoing
+# CM5 → peer traffic only.
+
+netem_init() {
+    [[ "$HAS_TC" -eq 1 && "$BIDI" -eq 1 ]] || return 0
+    # Load the ifb module (idempotent on modern kernels; ignored if built in).
+    sudo modprobe ifb numifbs=1 2>/dev/null || true
+    if ! ip link show "$IFB_DEV" >/dev/null 2>&1; then
+        fail "ifb device $IFB_DEV not present after modprobe; BIDI mode unavailable"
+        echo "  - check 'modprobe ifb' on this kernel, or run with BIDI=0"
+        return 1
+    fi
+    sudo ip link set "$IFB_DEV" up
+}
+
+netem_clear() {
+    [[ "$HAS_TC" -eq 1 ]] || return 0
+    sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
+    sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true
+    if [[ "$BIDI" -eq 1 ]]; then
+        sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true
+    fi
+}
+
+# Apply $1% loss in both directions (or egress-only when BIDI=0).
+netem_apply() {
+    local pct="$1"
+    [[ "$HAS_TC" -eq 1 ]] || return 0
+    netem_clear
+    [[ "$pct" -gt 0 ]] || return 0
+    # Egress: outgoing from $IFACE.
+    sudo tc qdisc add dev "$IFACE" root netem loss "${pct}%" 2>/dev/null || {
+        echo "Warning: failed to apply egress netem loss on $IFACE."
+        return 1
+    }
+    if [[ "$BIDI" -eq 1 ]]; then
+        # Ingress: incoming on $IFACE, redirected to $IFB_DEV egress and shaped there.
+        sudo tc qdisc add dev "$IFACE" handle ffff: ingress 2>/dev/null || {
+            echo "Warning: failed to add ingress qdisc on $IFACE."
+            return 1
+        }
+        sudo tc filter add dev "$IFACE" parent ffff: protocol all u32 \
+            match u32 0 0 action mirred egress redirect dev "$IFB_DEV" 2>/dev/null || {
+            echo "Warning: failed to install ingress redirect filter."
+            return 1
+        }
+        sudo tc qdisc add dev "$IFB_DEV" root netem loss "${pct}%" 2>/dev/null || {
+            echo "Warning: failed to apply netem on $IFB_DEV."
+            return 1
+        }
+    fi
+}
+
+netem_init || true
+
 step "Building ($BUILD)"
 if [[ "$BUILD" == "release" ]]; then
     if [[ "$RUN_SIMULATOR" -eq 1 ]]; then
@@ -84,9 +149,7 @@ done
 cleanup() {
     [[ -n "${SIM_PID:-}" ]] && kill -TERM "$SIM_PID" 2>/dev/null || true
     [[ -n "${SUBSTRATE_PID:-}" ]] && kill -TERM "$SUBSTRATE_PID" 2>/dev/null || true
-    if [[ "$HAS_TC" -eq 1 ]]; then
+    netem_clear
-        sudo tc qdisc del dev $IFACE root 2>/dev/null || true
-    fi
     wait 2>/dev/null || true
 }
 trap cleanup EXIT INT TERM
@@ -115,15 +178,8 @@ for entities in "${ENTITIES_LIST[@]}"; do
     devices=$(( entities / 7 ))
     
     for loss in "${LOSS_LIST[@]}"; do
-        # Apply tc netem loss
+        # Apply tc-netem loss in both directions (or egress-only when BIDI=0).
-        if [[ "$HAS_TC" -eq 1 ]]; then
+        netem_apply "$loss"
-            sudo tc qdisc del dev $IFACE root 2>/dev/null || true
-            if [[ "$loss" -gt 0 ]]; then
-                sudo tc qdisc add dev $IFACE root netem loss ${loss}% 2>/dev/null || {
-                    echo "Warning: failed to apply tc netem loss on interface $IFACE."
-                }
-            fi
-        fi
 
         if [[ "$RUN_SIMULATOR" -eq 1 ]]; then
             sim_args=(
@@ -178,8 +234,6 @@ for entities in "${ENTITIES_LIST[@]}"; do
     done
 done
 
-if [[ "$HAS_TC" -eq 1 ]]; then
+netem_clear
-    sudo tc qdisc del dev $IFACE root 2>/dev/null || true
-fi
 
 printf '\n%sCSV written to:%s %s\n' "$DIM" "$RESET" "$OUT_CSV"

scripts/verify-netem.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # scripts/verify-netem.sh — confirm tc-netem is actually applying loss, in
-# the direction you think it is.
+# the direction(s) you think it is.
 #
 # Usage:
 #   ./scripts/verify-netem.sh <peer-ip> [interface] [loss_pct]
@@ -10,26 +10,32 @@
 #   interface   Interface tc-netem is applied to. Default: eth0.
 #   loss_pct    Loss percentage to apply. Default: 5.
 #
+# Modes (env var):
+#   BIDI=0  (default)   Egress-only. Shapes outgoing traffic from <interface>.
+#                       Use a single ping from this host to verify.
+#   BIDI=1              Bidirectional via ifb ingress redirect. Shapes BOTH
+#                       outgoing AND incoming traffic on <interface>. Pings
+#                       run from THIS host verify egress; the script also
+#                       prompts you to ping back FROM THE PEER to verify
+#                       ingress (the script holds the qdisc up until you press
+#                       Enter, then tears everything down).
+#
 # What it does:
 #   1. Prints the current qdisc state (sanity check before).
-#   2. Applies `netem loss <loss_pct>%` on egress of <interface>.
+#   2. Applies the configured netem loss (egress, or egress + ingress).
 #   3. Re-prints the qdisc state (confirms the rule is installed).
 #   4. Sends 100 ICMP echo requests to <peer-ip> and reports the observed loss.
-#   5. Removes the qdisc on exit (trap), even on Ctrl-C.
+#   5. BIDI=1 only: waits for you to run `ping -c 100 <this-host>` from the
-#
+#      peer machine and report what it saw.
-# IMPORTANT direction caveat:
+#   6. Removes the qdiscs (and brings ifb0 down) on exit, even on Ctrl-C.
-#   tc qdisc rules apply to EGRESS only. If you run this on the CM5, it shapes
-#   CM5 → peer traffic. The peer → CM5 direction is unaffected. To drop traffic
-#   coming INTO this machine, you must either:
-#     (a) run netem on the SENDER side, or
-#     (b) attach an ifb (intermediate functional block) device for ingress.
-#   See the "Bidirectional loss" section in the script footer for the ifb recipe.
 
 set -euo pipefail
 
 PEER="${1:-}"
 IFACE="${2:-eth0}"
 LOSS="${3:-5}"
+BIDI="${BIDI:-0}"
+IFB_DEV="${IFB_DEV:-ifb0}"
 
 if [[ -z "$PEER" ]]; then
     echo "Usage: $0 <peer-ip> [interface] [loss_pct]"
@@ -69,23 +75,57 @@ fi
 step "Current qdisc on $IFACE (before)"
 sudo tc qdisc show dev "$IFACE" | sed 's/^/  /'
 
+# If bidi, ensure ifb device is up before installing qdiscs.
+if [[ "$BIDI" -eq 1 ]]; then
+    step "Bringing up $IFB_DEV (BIDI mode)"
+    sudo modprobe ifb numifbs=1 2>/dev/null || true
+    if ! ip link show "$IFB_DEV" >/dev/null 2>&1; then
+        fail "ifb device $IFB_DEV not present after modprobe; cannot run BIDI mode"
+        exit 1
+    fi
+    sudo ip link set "$IFB_DEV" up
+    ok "$IFB_DEV is up"
+fi
+
 # Apply netem
-step "Applying netem loss ${LOSS}% on $IFACE (egress)"
+if [[ "$BIDI" -eq 1 ]]; then
+    step "Applying netem loss ${LOSS}% on $IFACE (egress + ingress via $IFB_DEV)"
+else
+    step "Applying netem loss ${LOSS}% on $IFACE (egress only)"
+fi
 sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
+sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true
+[[ "$BIDI" -eq 1 ]] && sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true
 sudo tc qdisc add dev "$IFACE" root netem loss "${LOSS}%"
-ok "qdisc installed"
+if [[ "$BIDI" -eq 1 ]]; then
+    sudo tc qdisc add dev "$IFACE" handle ffff: ingress
+    sudo tc filter add dev "$IFACE" parent ffff: protocol all u32 \
+        match u32 0 0 action mirred egress redirect dev "$IFB_DEV"
+    sudo tc qdisc add dev "$IFB_DEV" root netem loss "${LOSS}%"
+fi
+ok "qdisc(s) installed"
 
 # Trap to clean up on any exit path
 cleanup() {
-    step "Removing netem qdisc from $IFACE"
+    step "Removing netem qdiscs"
     sudo tc qdisc del dev "$IFACE" root 2>/dev/null || true
-    ok "qdisc removed; $IFACE back to default"
+    sudo tc qdisc del dev "$IFACE" ingress 2>/dev/null || true
+    if [[ "$BIDI" -eq 1 ]]; then
+        sudo tc qdisc del dev "$IFB_DEV" root 2>/dev/null || true
+        sudo ip link set "$IFB_DEV" down 2>/dev/null || true
+    fi
+    ok "qdiscs removed; $IFACE back to default"
 }
 trap cleanup EXIT INT TERM
 
 # State AFTER install
-step "Current qdisc on $IFACE (after netem applied)"
+step "Current qdisc state (after netem applied)"
 sudo tc qdisc show dev "$IFACE" | sed 's/^/  /'
+if [[ "$BIDI" -eq 1 ]]; then
+    sudo tc qdisc show dev "$IFB_DEV" | sed 's/^/  /'
+    echo "  filter on $IFACE ingress:"
+    sudo tc filter show dev "$IFACE" parent ffff: 2>/dev/null | sed 's/^/    /'
+fi
 
 # Ping the peer and parse loss
 step "Pinging $PEER with 100 echoes (egress goes through netem)"
@@ -112,24 +152,17 @@ else
     warn "build doesn't include the loss module. Check 'modprobe sch_netem'."
 fi
 
-# ---------------------------------------------------------------------------
+if [[ "$BIDI" -eq 1 ]]; then
-# Bidirectional loss (info — not run by this script)
+    step "Now verify the INGRESS direction"
-# ---------------------------------------------------------------------------
+    THIS_IP="$(ip -4 addr show dev "$IFACE" | awk '/inet / {print $2}' | cut -d/ -f1 | head -1)"
-# tc qdisc shapes egress. To drop INCOMING traffic on the CM5 (e.g. T1
+    cat <<EOF
-# datagrams flowing from the Mac), redirect ingress to an ifb device and
+  From the peer machine, run:
-# shape that:
+    ping -c 100 -i 0.05 ${THIS_IP:-<this host\'s IP on $IFACE>}
-#
+  You should see ~${LOSS}% packet loss in the peer's ping output. That confirms
-#   sudo modprobe ifb numifbs=1
+  ingress shaping is dropping packets arriving here on $IFACE.
-#   sudo ip link set ifb0 up
+
-#   sudo tc qdisc add dev $IFACE handle ffff: ingress
+  Press Enter when done to tear everything down.
-#   sudo tc filter add dev $IFACE parent ffff: protocol all u32 \
+EOF
-#       match u32 0 0 action mirred egress redirect dev ifb0
+    read -r _ </dev/tty || true
-#   sudo tc qdisc add dev ifb0 root netem loss 5%
+fi
-#
+
-# Clean up:
-#   sudo tc qdisc del dev $IFACE ingress 2>/dev/null
-#   sudo tc qdisc del dev ifb0 root 2>/dev/null
-#   sudo ip link set ifb0 down
-#
-# Or — simpler — apply netem on BOTH ends with their respective egress qdiscs
-# (run this script with the same loss_pct on the Mac too).