Initial commit
This commit is contained in:
Valère Plantevin
265
run_benchmark.sh
Executable file
265
run_benchmark.sh
Executable file
@@ -0,0 +1,265 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================
|
||||
# ECS Digital Twin Benchmark — build + full sweep
|
||||
# TRANSIT lab / UQAC
|
||||
#
|
||||
# Detects the host CPU, builds with native hardware optimizations,
|
||||
# then runs the full entity sweep for the paper tables.
|
||||
#
|
||||
# Usage:
|
||||
# chmod +x run_benchmark.sh
|
||||
# ./run_benchmark.sh
|
||||
#
|
||||
# Output:
|
||||
# results/summary.txt — human-readable full log
|
||||
# results/csv/ — one CSV per entity count (for paper tables)
|
||||
# results/final_table.csv — aggregated table ready for LaTeX
|
||||
# =============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Paths ─────────────────────────────────────────────────────
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RESULTS_DIR="${SCRIPT_DIR}/results"
|
||||
CSV_DIR="${RESULTS_DIR}/csv"
|
||||
BINARY="${SCRIPT_DIR}/target/release/ecs_dt_benchmark"
|
||||
|
||||
mkdir -p "${RESULTS_DIR}" "${CSV_DIR}"
|
||||
|
||||
# ── Benchmark parameters ───────────────────────────────────────
|
||||
ENTITY_COUNTS=(10000 25000 50000 75000 100000 150000 200000)
|
||||
TICKS=5000
|
||||
WARMUP=500
|
||||
FAULT_PROB=0.02 # 2 % per tick per entity
|
||||
SEED=17446744073709551615
|
||||
|
||||
# ── Platform detection ────────────────────────────────────────
|
||||
ARCH="$(uname -m)"
|
||||
OS="$(uname -s)"
|
||||
|
||||
echo "============================================================"
|
||||
echo " ECS Digital Twin Benchmark — TRANSIT lab / UQAC"
|
||||
echo " Platform: ${OS} / ${ARCH}"
|
||||
echo "============================================================"
|
||||
|
||||
# Detect the best target-cpu flag for this machine.
|
||||
detect_target_cpu() {
|
||||
case "${ARCH}" in
|
||||
aarch64)
|
||||
# On RPi 5 (Cortex-A76) /proc/cpuinfo lists "CPU part : 0xd0b"
|
||||
# On Apple M-series (native aarch64 via Rosetta or native build)
|
||||
# uname gives arm64 on macOS, aarch64 on Linux.
|
||||
if [[ "${OS}" == "Darwin" ]]; then
|
||||
# Apple Silicon — the toolchain knows the exact µarch.
|
||||
echo "apple-m1"
|
||||
elif grep -q "Cortex-A76" /proc/cpuinfo 2>/dev/null || \
|
||||
grep -q "0xd0b" /proc/cpuinfo 2>/dev/null; then
|
||||
echo "cortex-a76" # RPi 5
|
||||
elif grep -q "Cortex-A72" /proc/cpuinfo 2>/dev/null || \
|
||||
grep -q "0xd08" /proc/cpuinfo 2>/dev/null; then
|
||||
echo "cortex-a72" # RPi 4
|
||||
else
|
||||
echo "native" # fall back: let LLVM auto-detect
|
||||
fi
|
||||
;;
|
||||
x86_64)
|
||||
echo "native" # covers Ryzen, Intel, any x86-64
|
||||
;;
|
||||
arm64)
|
||||
# macOS reports arm64
|
||||
echo "apple-m1"
|
||||
;;
|
||||
*)
|
||||
echo "native"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
TARGET_CPU="$(detect_target_cpu)"
|
||||
echo " Target CPU flag: -C target-cpu=${TARGET_CPU}"
|
||||
|
||||
# ── CPU isolation advice ───────────────────────────────────────
|
||||
# On Linux, pin to an isolated core if isolcpus was set at boot.
|
||||
# On the RPi 5 for the paper we recommend:
|
||||
# sudo systemctl isolate multi-user.target (drop GUI)
|
||||
# TASKSET="taskset -c 0"
|
||||
# We auto-detect isolated cores from the kernel cmdline.
|
||||
TASKSET=""
|
||||
if [[ "${OS}" == "Linux" ]]; then
|
||||
ISOLATED=""
|
||||
if [[ -r /sys/devices/system/cpu/isolated ]]; then
|
||||
ISOLATED="$(cat /sys/devices/system/cpu/isolated)"
|
||||
fi
|
||||
if [[ -n "${ISOLATED}" ]]; then
|
||||
# Use the first isolated core.
|
||||
FIRST_CORE="${ISOLATED%%[-,]*}"
|
||||
TASKSET="taskset -c ${FIRST_CORE}"
|
||||
echo " Isolated cores: ${ISOLATED} → pinning to core ${FIRST_CORE}"
|
||||
else
|
||||
echo " No isolated cores detected — running unpinned."
|
||||
echo " Tip: add isolcpus=2 nohz_full=2 rcu_nocbs=2 to /boot/cmdline.txt"
|
||||
echo " for more stable single-core measurements on RPi 5."
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Build ──────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "── Building (release + native optimizations) ────────────"
|
||||
|
||||
RUSTFLAGS="-C target-cpu=${TARGET_CPU} -C opt-level=3" \
|
||||
cargo build --release 2>&1 | tail -5
|
||||
|
||||
echo " Binary: ${BINARY}"
|
||||
echo " Size: $(du -sh "${BINARY}" | cut -f1)"
|
||||
|
||||
# ── System info snapshot ───────────────────────────────────────
|
||||
SYSINFO_FILE="${RESULTS_DIR}/sysinfo.txt"
|
||||
{
|
||||
echo "=== System Information ==="
|
||||
echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
||||
echo "OS: ${OS} $(uname -r 2>/dev/null || true)"
|
||||
echo "Arch: ${ARCH}"
|
||||
echo "Target CPU: ${TARGET_CPU}"
|
||||
echo "Rust: $(rustc --version)"
|
||||
echo "Cargo: $(cargo --version)"
|
||||
echo ""
|
||||
if [[ "${OS}" == "Linux" ]]; then
|
||||
echo "=== CPU Info ==="
|
||||
grep -E "^(model name|Hardware|CPU part|CPU implementer|Revision)" \
|
||||
/proc/cpuinfo 2>/dev/null | sort -u || true
|
||||
echo ""
|
||||
echo "=== Memory ==="
|
||||
grep -E "^(MemTotal|MemAvailable)" /proc/meminfo 2>/dev/null || true
|
||||
echo ""
|
||||
echo "=== CPU Governor ==="
|
||||
for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
[[ -r "$f" ]] && echo " $f: $(cat "$f")" && break
|
||||
done
|
||||
echo ""
|
||||
echo "=== Isolated cores ==="
|
||||
cat /sys/devices/system/cpu/isolated 2>/dev/null || echo " none"
|
||||
elif [[ "${OS}" == "Darwin" ]]; then
|
||||
sysctl -n machdep.cpu.brand_string 2>/dev/null || true
|
||||
sysctl -n hw.memsize 2>/dev/null | \
|
||||
awk '{printf "Memory: %.1f GB\n", $1/1073741824}' || true
|
||||
fi
|
||||
} | tee "${SYSINFO_FILE}"
|
||||
|
||||
echo ""
|
||||
|
||||
# ── CSV header ────────────────────────────────────────────────
|
||||
FINAL_CSV="${RESULTS_DIR}/final_table.csv"
|
||||
echo "entities,ticks,wall_s,hz,per_tick_us,ingest_us,sim_us,export_us,diag_us,\
|
||||
sensor_faults,actuator_lockouts,exported_mb,acks,alerts,rss_mb" \
|
||||
> "${FINAL_CSV}"
|
||||
|
||||
# ── Sweep ─────────────────────────────────────────────────────
|
||||
echo "── Running entity sweep ─────────────────────────────────"
|
||||
echo " Counts: ${ENTITY_COUNTS[*]}"
|
||||
echo " Ticks: ${TICKS} Warmup: ${WARMUP} Fault: ${FAULT_PROB}"
|
||||
echo ""
|
||||
|
||||
SUMMARY_FILE="${RESULTS_DIR}/summary.txt"
|
||||
echo "ECS DT Benchmark Sweep — $(date -u)" > "${SUMMARY_FILE}"
|
||||
cat "${SYSINFO_FILE}" >> "${SUMMARY_FILE}"
|
||||
echo "" >> "${SUMMARY_FILE}"
|
||||
|
||||
for N in "${ENTITY_COUNTS[@]}"; do
|
||||
echo "── entities=${N} ─────────────────────────────────────"
|
||||
|
||||
RAW_FILE="${RESULTS_DIR}/raw_${N}.txt"
|
||||
|
||||
# Run benchmark, tee to file and stdout.
|
||||
# shellcheck disable=SC2086
|
||||
${TASKSET} "${BINARY}" \
|
||||
--entities "${N}" \
|
||||
--ticks "${TICKS}" \
|
||||
--seed "${SEED}" \
|
||||
--fault-prob "${FAULT_PROB}" \
|
||||
2>&1 | tee "${RAW_FILE}"
|
||||
|
||||
echo "" >> "${SUMMARY_FILE}"
|
||||
echo "=== entities=${N} ===" >> "${SUMMARY_FILE}"
|
||||
cat "${RAW_FILE}" >> "${SUMMARY_FILE}"
|
||||
|
||||
# ── Parse the Final Summary block from the binary output ──
|
||||
# Extract values using grep + awk on the known output format.
|
||||
parse() {
|
||||
grep -m1 "$1" "${RAW_FILE}" | awk '{print $NF}'
|
||||
}
|
||||
|
||||
WALL_S=$(grep "Total wall time" "${RAW_FILE}" | awk '{print $NF}' | tr -d 's')
|
||||
HZ=$(grep "Sustained tick rate" "${RAW_FILE}" | awk '{print $NF}')
|
||||
PTW=$(grep "Per-tick mean" "${RAW_FILE}" | awk '{print $NF}')
|
||||
INGEST=$(grep "IngestSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}')
|
||||
SIM=$(grep "SimulationSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}')
|
||||
EXPORT=$(grep "ExportSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}')
|
||||
DIAGSYS=$(grep "DiagnosticsSystem:" "${RAW_FILE}" | tail -1 | awk '{print $NF}')
|
||||
FAULTS=$(grep "Total sensor faults:" "${RAW_FILE}" | awk '{print $NF}')
|
||||
LOCKOUTS=$(grep "Total actuator lockouts" "${RAW_FILE}" | awk '{print $NF}')
|
||||
EXPMT=$(grep "Bytes exported:" "${RAW_FILE}" | awk '{print $NF}')
|
||||
ACKS=$(grep "Acks processed:" "${RAW_FILE}" | awk '{print $NF}')
|
||||
ALTS=$(grep "Alerts seen:" "${RAW_FILE}" | awk '{print $NF}')
|
||||
RSS=$(grep "Memory (RSS" "${RAW_FILE}" | awk '{print $NF}')
|
||||
|
||||
# Remove trailing units that awk picked up (MB, Hz, µs, s).
|
||||
strip() { echo "$1" | tr -d 'MBHzµs '; }
|
||||
|
||||
echo "${N},${TICKS},$(strip "$WALL_S"),$(strip "$HZ"),$(strip "$PTW"),\
|
||||
$(strip "$INGEST"),$(strip "$SIM"),$(strip "$EXPORT"),$(strip "$DIAGSYS"),\
|
||||
$(strip "$FAULTS"),$(strip "$LOCKOUTS"),$(strip "$EXPMT"),\
|
||||
$(strip "$ACKS"),$(strip "$ALTS"),$(strip "$RSS")" \
|
||||
>> "${FINAL_CSV}"
|
||||
|
||||
# Per-entity CSV for detailed analysis.
|
||||
cat "${RAW_FILE}" > "${CSV_DIR}/entities_${N}.txt"
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ── Pretty-print final table ──────────────────────────────────
|
||||
echo "============================================================"
|
||||
echo " Final Results Table"
|
||||
echo "============================================================"
|
||||
echo ""
|
||||
printf "%-10s %-8s %-10s %-12s %-12s %-10s %-10s\n" \
|
||||
"Entities" "Hz" "Tick(µs)" "Ingest(µs)" "Sim(µs)" "Export(µs)" "RSS(MB)"
|
||||
printf "%-10s %-8s %-10s %-12s %-12s %-10s %-10s\n" \
|
||||
"--------" "------" "---------" "----------" "--------" "----------" "-------"
|
||||
|
||||
tail -n +2 "${FINAL_CSV}" | while IFS=',' read -r \
|
||||
entities ticks wall hz ptw ingest sim export diag \
|
||||
faults lockouts expmt acks alts rss; do
|
||||
printf "%-10s %-8s %-10s %-12s %-12s %-10s %-10s\n" \
|
||||
"${entities}" "${hz}" "${ptw}" "${ingest}" "${sim}" "${export}" "${rss}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "── Output files ─────────────────────────────────────────"
|
||||
echo " ${SUMMARY_FILE}"
|
||||
echo " ${FINAL_CSV}"
|
||||
echo " ${CSV_DIR}/"
|
||||
echo ""
|
||||
echo "── LaTeX snippet for paper Table 2 ─────────────────────"
|
||||
echo '\begin{table}[h]'
|
||||
echo '\caption{ECS DT Runtime: Memory and Throughput Scaling (RPi~5, single core)}'
|
||||
echo '\centering\renewcommand{\arraystretch}{1.25}'
|
||||
echo '\begin{tabular}{@{}rrrrr@{}}'
|
||||
echo '\toprule'
|
||||
echo '\textbf{Entities} & \textbf{RSS (MB)} & \textbf{Tick rate (Hz)} &'
|
||||
echo '\textbf{Per-tick ($\mu$s)} & \textbf{Sim system ($\mu$s)} \\'
|
||||
echo '\midrule'
|
||||
|
||||
tail -n +2 "${FINAL_CSV}" | while IFS=',' read -r \
|
||||
entities ticks wall hz ptw ingest sim export diag \
|
||||
faults lockouts expmt acks alts rss; do
|
||||
printf '%s & %s & %s & %s & %s \\\\\n' \
|
||||
"$(printf '%s' "${entities}" | sed 's/000$/k/' | sed 's/00k$/00k/')" \
|
||||
"${rss}" "${hz}" "${ptw}" "${sim}"
|
||||
done
|
||||
|
||||
echo '\bottomrule'
|
||||
echo '\end{tabular}'
|
||||
echo '\end{table}'
|
||||
echo ""
|
||||
echo "Done. $(date -u)"
|
||||
Reference in New Issue
Block a user