#!/usr/bin/env bash # ============================================================= # ECS Digital Twin Benchmark — build + full sweep # TRANSIT lab / UQAC # # Detects the host CPU, builds with native hardware optimizations, # then runs the full entity sweep for the paper tables. # # Usage: # chmod +x run_benchmark.sh # ./run_benchmark.sh # # Output: # results/summary.txt — human-readable full log # results/csv/ — one CSV per entity count (for paper tables) # results/final_table.csv — aggregated table ready for LaTeX # ============================================================= set -euo pipefail # ── Paths ───────────────────────────────────────────────────── SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RESULTS_DIR="${SCRIPT_DIR}/results" CSV_DIR="${RESULTS_DIR}/csv" BINARY="${SCRIPT_DIR}/target/release/ecs_dt_benchmark" mkdir -p "${RESULTS_DIR}" "${CSV_DIR}" # ── Benchmark parameters ─────────────────────────────────────── ENTITY_COUNTS=(10000 25000 50000 75000 100000 150000 200000) TICKS=5000 WARMUP=500 FAULT_PROB=0.02 # 2 % per tick per entity SEED=17446744073709551615 # ── Platform detection ──────────────────────────────────────── ARCH="$(uname -m)" OS="$(uname -s)" echo "============================================================" echo " ECS Digital Twin Benchmark — TRANSIT lab / UQAC" echo " Platform: ${OS} / ${ARCH}" echo "============================================================" # Detect the best target-cpu flag for this machine. detect_target_cpu() { case "${ARCH}" in aarch64) # On RPi 5 (Cortex-A76) /proc/cpuinfo lists "CPU part : 0xd0b" # On Apple M-series (native aarch64 via Rosetta or native build) # uname gives arm64 on macOS, aarch64 on Linux. if [[ "${OS}" == "Darwin" ]]; then # Apple Silicon — the toolchain knows the exact µarch. echo "apple-m1" elif grep -q "Cortex-A76" /proc/cpuinfo 2>/dev/null || \ grep -q "0xd0b" /proc/cpuinfo 2>/dev/null; then echo "cortex-a76" # RPi 5 elif grep -q "Cortex-A72" /proc/cpuinfo 2>/dev/null || \ grep -q "0xd08" /proc/cpuinfo 2>/dev/null; then echo "cortex-a72" # RPi 4 else echo "native" # fall back: let LLVM auto-detect fi ;; x86_64) echo "native" # covers Ryzen, Intel, any x86-64 ;; arm64) # macOS reports arm64 echo "apple-m1" ;; *) echo "native" ;; esac } TARGET_CPU="$(detect_target_cpu)" echo " Target CPU flag: -C target-cpu=${TARGET_CPU}" # ── CPU isolation advice ─────────────────────────────────────── # On Linux, pin to an isolated core if isolcpus was set at boot. # On the RPi 5 for the paper we recommend: # sudo systemctl isolate multi-user.target (drop GUI) # TASKSET="taskset -c 0" # We auto-detect isolated cores from the kernel cmdline. TASKSET="" if [[ "${OS}" == "Linux" ]]; then ISOLATED="" if [[ -r /sys/devices/system/cpu/isolated ]]; then ISOLATED="$(cat /sys/devices/system/cpu/isolated)" fi if [[ -n "${ISOLATED}" ]]; then # Use the first isolated core. FIRST_CORE="${ISOLATED%%[-,]*}" TASKSET="taskset -c ${FIRST_CORE}" echo " Isolated cores: ${ISOLATED} → pinning to core ${FIRST_CORE}" else echo " No isolated cores detected — running unpinned." echo " Tip: add isolcpus=2 nohz_full=2 rcu_nocbs=2 to /boot/cmdline.txt" echo " for more stable single-core measurements on RPi 5." fi fi # ── Build ────────────────────────────────────────────────────── echo "" echo "── Building (release + native optimizations) ────────────" RUSTFLAGS="-C target-cpu=${TARGET_CPU} -C opt-level=3" \ cargo build --release 2>&1 | tail -5 echo " Binary: ${BINARY}" echo " Size: $(du -sh "${BINARY}" | cut -f1)" # ── System info snapshot ─────────────────────────────────────── SYSINFO_FILE="${RESULTS_DIR}/sysinfo.txt" { echo "=== System Information ===" echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" echo "OS: ${OS} $(uname -r 2>/dev/null || true)" echo "Arch: ${ARCH}" echo "Target CPU: ${TARGET_CPU}" echo "Rust: $(rustc --version)" echo "Cargo: $(cargo --version)" echo "" if [[ "${OS}" == "Linux" ]]; then echo "=== CPU Info ===" grep -E "^(model name|Hardware|CPU part|CPU implementer|Revision)" \ /proc/cpuinfo 2>/dev/null | sort -u || true echo "" echo "=== Memory ===" grep -E "^(MemTotal|MemAvailable)" /proc/meminfo 2>/dev/null || true echo "" echo "=== CPU Governor ===" for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do [[ -r "$f" ]] && echo " $f: $(cat "$f")" && break done echo "" echo "=== Isolated cores ===" cat /sys/devices/system/cpu/isolated 2>/dev/null || echo " none" elif [[ "${OS}" == "Darwin" ]]; then sysctl -n machdep.cpu.brand_string 2>/dev/null || true sysctl -n hw.memsize 2>/dev/null | \ awk '{printf "Memory: %.1f GB\n", $1/1073741824}' || true fi } | tee "${SYSINFO_FILE}" echo "" # ── CSV header ──────────────────────────────────────────────── FINAL_CSV="${RESULTS_DIR}/final_table.csv" echo "entities,ticks,wall_s,hz,per_tick_us,ingest_us,sim_us,export_us,diag_us,\ sensor_faults,actuator_lockouts,exported_mb,acks,alerts,rss_mb" \ > "${FINAL_CSV}" # ── Sweep ───────────────────────────────────────────────────── echo "── Running entity sweep ─────────────────────────────────" echo " Counts: ${ENTITY_COUNTS[*]}" echo " Ticks: ${TICKS} Warmup: ${WARMUP} Fault: ${FAULT_PROB}" echo "" SUMMARY_FILE="${RESULTS_DIR}/summary.txt" echo "ECS DT Benchmark Sweep — $(date -u)" > "${SUMMARY_FILE}" cat "${SYSINFO_FILE}" >> "${SUMMARY_FILE}" echo "" >> "${SUMMARY_FILE}" for N in "${ENTITY_COUNTS[@]}"; do echo "── entities=${N} ─────────────────────────────────────" RAW_FILE="${RESULTS_DIR}/raw_${N}.txt" # Run benchmark, tee to file and stdout. # shellcheck disable=SC2086 ${TASKSET} "${BINARY}" \ --entities "${N}" \ --ticks "${TICKS}" \ --seed "${SEED}" \ --fault-prob "${FAULT_PROB}" \ 2>&1 | tee "${RAW_FILE}" echo "" >> "${SUMMARY_FILE}" echo "=== entities=${N} ===" >> "${SUMMARY_FILE}" cat "${RAW_FILE}" >> "${SUMMARY_FILE}" # ── Parse the Final Summary block from the binary output ── # Extract values using grep + awk on the known output format. parse() { grep -m1 "$1" "${RAW_FILE}" | awk '{print $NF}' } WALL_S=$(grep "Total wall time" "${RAW_FILE}" | awk '{print $NF}' | tr -d 's') HZ=$(grep "Sustained tick rate" "${RAW_FILE}" | awk '{print $NF}') PTW=$(grep "Per-tick mean" "${RAW_FILE}" | awk '{print $NF}') INGEST=$(grep "IngestSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}') SIM=$(grep "SimulationSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}') EXPORT=$(grep "ExportSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}') DIAGSYS=$(grep "DiagnosticsSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}') FAULTS=$(grep "Total sensor faults:" "${RAW_FILE}" | awk '{print $NF}') LOCKOUTS=$(grep "Total actuator lockouts" "${RAW_FILE}" | awk '{print $NF}') EXPMT=$(grep "Bytes exported:" "${RAW_FILE}" | awk '{print $NF}') ACKS=$(grep "Acks processed:" "${RAW_FILE}" | awk '{print $NF}') ALTS=$(grep "Alerts seen:" "${RAW_FILE}" | awk '{print $NF}') RSS=$(grep "Memory (RSS" "${RAW_FILE}" | awk '{print $NF}') # Remove trailing units that awk picked up (MB, Hz, µs, s). strip() { echo "$1" | tr -d 'MBHzµs '; } echo "${N},${TICKS},$(strip "$WALL_S"),$(strip "$HZ"),$(strip "$PTW"),\ $(strip "$INGEST"),$(strip "$SIM"),$(strip "$EXPORT"),$(strip "$DIAGSYS"),\ $(strip "$FAULTS"),$(strip "$LOCKOUTS"),$(strip "$EXPMT"),\ $(strip "$ACKS"),$(strip "$ALTS"),$(strip "$RSS")" \ >> "${FINAL_CSV}" # Per-entity CSV for detailed analysis. cat "${RAW_FILE}" > "${CSV_DIR}/entities_${N}.txt" echo "" done # ── Pretty-print final table ────────────────────────────────── echo "============================================================" echo " Final Results Table" echo "============================================================" echo "" printf "%-10s %-8s %-10s %-12s %-12s %-10s %-10s\n" \ "Entities" "Hz" "Tick(µs)" "Ingest(µs)" "Sim(µs)" "Export(µs)" "RSS(MB)" printf "%-10s %-8s %-10s %-12s %-12s %-10s %-10s\n" \ "--------" "------" "---------" "----------" "--------" "----------" "-------" tail -n +2 "${FINAL_CSV}" | while IFS=',' read -r \ entities ticks wall hz ptw ingest sim export diag \ faults lockouts expmt acks alts rss; do printf "%-10s %-8s %-10s %-12s %-12s %-10s %-10s\n" \ "${entities}" "${hz}" "${ptw}" "${ingest}" "${sim}" "${export}" "${rss}" done echo "" echo "── Output files ─────────────────────────────────────────" echo " ${SUMMARY_FILE}" echo " ${FINAL_CSV}" echo " ${CSV_DIR}/" echo "" echo "── LaTeX snippet for paper Table 2 ─────────────────────" echo '\begin{table}[h]' echo '\caption{ECS DT Runtime: Memory and Throughput Scaling (RPi~5, single core)}' echo '\centering\renewcommand{\arraystretch}{1.25}' echo '\begin{tabular}{@{}rrrrr@{}}' echo '\toprule' echo '\textbf{Entities} & \textbf{RSS (MB)} & \textbf{Tick rate (Hz)} &' echo '\textbf{Per-tick ($\mu$s)} & \textbf{Sim system ($\mu$s)} \\' echo '\midrule' tail -n +2 "${FINAL_CSV}" | while IFS=',' read -r \ entities ticks wall hz ptw ingest sim export diag \ faults lockouts expmt acks alts rss; do printf '%s & %s & %s & %s & %s \\\\\n' \ "$(printf '%s' "${entities}" | sed 's/000$/k/' | sed 's/00k$/00k/')" \ "${rss}" "${hz}" "${ptw}" "${sim}" done echo '\bottomrule' echo '\end{tabular}' echo '\end{table}' echo "" echo "Done. $(date -u)"