First test kinda working

This commit is contained in:
Valère Plantevin
2026-05-12 11:21:40 -04:00
parent cac6c9ac02
commit d3f09ee062
36 changed files with 3903 additions and 102 deletions

View File

@@ -0,0 +1,97 @@
//! Components attached to per-sensor entities, plus the per-type threshold
//! table used by `simulation_system`'s crossing detection.
//!
//! Each (device, sensor) pair becomes one entity tagged with `Asset` and
//! carrying `DeviceId` + `SensorId` + `SensorTypeTag` + `RawSensorData` +
//! `SmoothedValue`.
use bevy::prelude::*;
use crate::transport::SensorType;
/// Marker — every (device, sensor) pair becomes one entity tagged `Asset`.
#[derive(Component, Debug, Default, Clone, Copy)]
pub struct Asset;
#[derive(Component, Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DeviceId(pub uuid::Uuid);
#[derive(Component, Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SensorId(pub u16);
/// Sensor type — set on entity creation from the first message that names
/// the (device, sensor) pair, then immutable. We don't track type changes:
/// a given (device_id, sensor_id) is one logical sensor with one type for
/// the lifetime of the run.
#[derive(Component, Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SensorTypeTag(pub SensorType);
/// Latest reading from this (device, sensor). Updated in place by
/// `ingest_system`; read by simulation/export/diagnostics.
#[derive(Component, Debug, Default, Clone, Copy, PartialEq)]
pub struct RawSensorData {
pub raw_value: f64,
pub timestamp_us: u64,
pub sequence_number: u32,
}
pub const SMOOTHED_WINDOW: usize = 16;
/// Rolling-window mean of the last `SMOOTHED_WINDOW` raw readings, plus a
/// hysteresis flag for threshold-crossing detection. Maintained by
/// `simulation_system` — this is the bit of the ECS that does honest
/// digital-twin transform work, not just write-through of incoming samples.
#[derive(Component, Debug, Clone, Copy)]
pub struct SmoothedValue {
ring: [f64; SMOOTHED_WINDOW],
head: usize,
filled: u16,
pub mean: f64,
pub above_threshold: bool,
}
impl Default for SmoothedValue {
fn default() -> Self {
Self {
ring: [0.0; SMOOTHED_WINDOW],
head: 0,
filled: 0,
mean: 0.0,
above_threshold: false,
}
}
}
impl SmoothedValue {
/// Push a new sample. Non-finite values (NaN / ±∞) are ignored — the
/// smoothed state stays whatever it was. This matters because T3 acks
/// can carry NaN when the substrate has never seen the target sensor.
pub fn push(&mut self, v: f64) {
if !v.is_finite() {
return;
}
self.ring[self.head] = v;
self.head = (self.head + 1) % SMOOTHED_WINDOW;
if (self.filled as usize) < SMOOTHED_WINDOW {
self.filled += 1;
}
let n = self.filled as usize;
let sum: f64 = self.ring.iter().take(n).sum();
self.mean = sum / n as f64;
}
}
/// Per-type threshold for `simulation_system`'s crossing detection. Chosen
/// mid-band against the simulator's waveforms so crossings actually fire
/// during a demo; in a real deployment these would be alarm thresholds
/// supplied by config.
pub(super) fn threshold_for(t: SensorType) -> f64 {
match t {
SensorType::Generic => 0.0,
SensorType::Temperature => 22.0, // °C — simulator oscillates 15..25
SensorType::Humidity => 55.0, // % — 30..70
SensorType::Pressure => 1014.0, // hPa — 1008..1018
SensorType::Voltage => 230.2, // V — 229.5..230.5
SensorType::Current => 10.5, // A — 8..12
}
}

View File

@@ -0,0 +1,52 @@
//! ECS world: the five paper-named systems plus the components and resources
//! they operate on.
//!
//! ```text
//! components.rs ── per-sensor components + per-type threshold table
//! resources.rs ── SensorRegistry, DiagnosticsState, ExportSampleState
//! systems.rs ── ingest / fault_injection / simulation / export / diagnostics
//! tests.rs ── unit tests (#[cfg(test)] only)
//! ```
//!
//! Each (device, sensor) pair becomes one entity with `Asset` + `DeviceId` +
//! `SensorId` + `SensorTypeTag` + `RawSensorData` + `SmoothedValue`.
//! `ingest_system` upserts on every incoming `QuicMessage`; the registry maps
//! `(Uuid, u16) → Entity` for O(1) lookup.
mod components;
mod resources;
mod systems;
#[cfg(test)]
mod tests;
use bevy::prelude::*;
use bevy::state::condition::in_state;
use crate::transport::state::ServerState;
pub use components::{
Asset, DeviceId, RawSensorData, SMOOTHED_WINDOW, SensorId, SensorTypeTag, SmoothedValue,
};
pub use resources::SensorRegistry;
pub struct WorldPlugin;
impl Plugin for WorldPlugin {
fn build(&self, app: &mut App) {
app.init_resource::<SensorRegistry>()
.init_resource::<resources::DiagnosticsState>()
.init_resource::<resources::ExportSampleState>()
.add_systems(
PreUpdate,
(systems::fault_injection_system, systems::ingest_system)
.chain()
.run_if(in_state(ServerState::Started)),
)
.add_systems(Update, systems::simulation_system)
.add_systems(
PostUpdate,
(systems::export_system, systems::diagnostics_system).chain(),
);
}
}

View File

@@ -0,0 +1,48 @@
//! Bevy `Resource`s consumed by the world's systems.
use std::collections::HashMap;
use std::time::Instant;
use bevy::prelude::{Entity, Resource};
/// O(1) lookup `(device_id, sensor_id) → Entity`. Populated lazily by the
/// ingest system; queried by export/diagnostics.
#[derive(Resource, Default)]
pub struct SensorRegistry {
pub(crate) map: HashMap<(uuid::Uuid, u16), Entity>,
}
impl SensorRegistry {
pub fn entity_count(&self) -> usize {
self.map.len()
}
}
/// Rolling counter of ticks since the last `diagnostics` log line was emitted.
#[derive(Resource)]
pub(super) struct DiagnosticsState {
pub(super) last_log: Instant,
pub(super) ticks_since_log: u64,
}
impl Default for DiagnosticsState {
fn default() -> Self {
Self {
last_log: Instant::now(),
ticks_since_log: 0,
}
}
}
/// Rate-limiter for `export_system` — runs at the ECS tick rate but only
/// emits gauges once per second.
#[derive(Resource)]
pub(super) struct ExportSampleState {
pub(super) last_sample: Instant,
}
impl Default for ExportSampleState {
fn default() -> Self {
Self { last_sample: Instant::now() }
}
}

View File

@@ -0,0 +1,278 @@
//! The five paper-named ECS systems and their private helpers.
//!
//! Scheduler placement (configured in [`super::WorldPlugin`]):
//!
//! | Schedule | Systems |
//! |-----------|--------------------------------------|
//! | PreUpdate | fault_injection → ingest |
//! | Update | simulation |
//! | PostUpdate| export → diagnostics |
use std::collections::HashMap;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use bevy::prelude::*;
use metrics::{counter, gauge, histogram};
use crate::transport::ecs::{BridgeReceivers, BridgeSenders};
use crate::transport::{QuicMessage, SensorType};
use super::components::{
Asset, DeviceId, RawSensorData, SensorId, SensorTypeTag, SmoothedValue, threshold_for,
};
use super::resources::{DiagnosticsState, ExportSampleState, SensorRegistry};
/// T1 batch limit per tick. Anything beyond this stays in the channel and
/// either drains next tick or gets dropped on full (T1's contract is lossy).
const T1_INGEST_BATCH: usize = 1024;
/// Drain the three tier channels into ECS state.
///
/// T1: bounded batch (lossy); T2: full drain (reliable); T3: full drain, with
/// each command answered by an ack carrying the device's current sensor value.
pub(super) fn ingest_system(
bridge: Res<BridgeReceivers>,
mut registry: ResMut<SensorRegistry>,
mut commands: Commands,
mut q: Query<&mut RawSensorData>,
) {
let now = now_us();
// T1 — datagrams.
{
let mut t1 = bridge.t1.lock().unwrap();
for _ in 0..T1_INGEST_BATCH {
match t1.try_recv() {
Ok(msg) => {
histogram!("substrate_latency_us", "tier" => "t1")
.record(now.saturating_sub(msg.timestamp_us) as f64);
upsert_reading(&mut registry, &mut commands, &mut q, msg);
}
Err(_) => break,
}
}
}
// T2 — uni streams.
{
let mut t2 = bridge.t2.lock().unwrap();
while let Ok(msg) = t2.try_recv() {
histogram!("substrate_latency_us", "tier" => "t2")
.record(now.saturating_sub(msg.timestamp_us) as f64);
upsert_reading(&mut registry, &mut commands, &mut q, msg);
}
}
// T3 — bidirectional commands. Reply with the device's most recent
// sensor value (NaN if we've never seen this (device, sensor) before).
{
let mut t3 = bridge.t3.lock().unwrap();
while let Ok(inbound) = t3.try_recv() {
histogram!("substrate_latency_us", "tier" => "t3")
.record(now.saturating_sub(inbound.command.timestamp_us) as f64);
let key = (inbound.command.device_id, inbound.command.sensor_id);
let current_value = registry
.map
.get(&key)
.and_then(|&e| q.get(e).ok())
.map(|d| d.raw_value)
.unwrap_or(f64::NAN);
let ack = QuicMessage {
device_id: inbound.command.device_id,
sensor_id: inbound.command.sensor_id,
raw_value: current_value,
timestamp_us: now_us(),
sequence_number: inbound.command.sequence_number,
sensor_type: inbound.command.sensor_type,
};
// Ignore send errors: the demux task may have given up if the
// connection died while we were processing.
let _ = inbound.reply.send(ack);
}
}
}
fn upsert_reading(
registry: &mut SensorRegistry,
commands: &mut Commands,
q: &mut Query<&mut RawSensorData>,
msg: QuicMessage,
) {
let key = (msg.device_id, msg.sensor_id);
let data = RawSensorData {
raw_value: msg.raw_value,
timestamp_us: msg.timestamp_us,
sequence_number: msg.sequence_number,
};
if let Some(&entity) = registry.map.get(&key) {
// Common case: existing entity, mutate in place.
if let Ok(mut existing) = q.get_mut(entity) {
*existing = data;
} else {
// Edge case: entity was registered earlier in *this* tick via
// `commands.spawn`, so the components aren't in the archetype
// yet (`Commands` is deferred). Queue another insert; last write
// wins when Commands flushes.
commands.entity(entity).insert(data);
}
return;
}
let entity = commands
.spawn((
Asset,
DeviceId(msg.device_id),
SensorId(msg.sensor_id),
SensorTypeTag(SensorType::from_u8(msg.sensor_type)),
SmoothedValue::default(),
data,
))
.id();
registry.map.insert(key, entity);
}
/// Stub — M6 inserts loss/delay here for benchmark scenarios.
pub(super) fn fault_injection_system() {}
/// Per-sensor digital-twin transform. Pulls each entity's latest
/// `RawSensorData` into a sliding-window mean (`SmoothedValue`), and emits
/// `substrate_threshold_crossings_total{type, direction}` when that mean
/// transitions across the per-type threshold. The `Changed<RawSensorData>`
/// filter restricts the scan to entities updated *this tick*, so the cost
/// scales with ingress rate, not fleet size.
pub(super) fn simulation_system(
mut q: Query<(&SensorTypeTag, &RawSensorData, &mut SmoothedValue), Changed<RawSensorData>>,
) {
for (st, raw, mut smoothed) in q.iter_mut() {
smoothed.push(raw.raw_value);
let now_above = smoothed.mean > threshold_for(st.0);
if now_above != smoothed.above_threshold {
smoothed.above_threshold = now_above;
let dir = if now_above { "up" } else { "down" };
counter!(
"substrate_threshold_crossings_total",
"type" => st.0.label_str(),
"direction" => dir
)
.increment(1);
}
}
}
/// Sample ECS-side gauges into the Prometheus exporter. Runs every tick but
/// only emits once per second to keep cost negligible. This is the system
/// the paper's §Architecture diagram calls `ExportSystem`.
pub(super) fn export_system(
senders: Res<BridgeSenders>,
registry: Res<SensorRegistry>,
sensors_q: Query<(&SensorTypeTag, &RawSensorData)>,
mut state: ResMut<ExportSampleState>,
) {
let now = Instant::now();
if now.duration_since(state.last_sample) < Duration::from_secs(1) {
return;
}
state.last_sample = now;
// ---- runtime telemetry ----
gauge!("substrate_entities").set(registry.entity_count() as f64);
gauge!("substrate_channel_depth", "tier" => "t1").set(senders.t1.depth() as f64);
gauge!("substrate_channel_depth", "tier" => "t2").set(senders.t2.depth() as f64);
gauge!("substrate_channel_depth", "tier" => "t3").set(senders.t3.depth() as f64);
gauge!("substrate_channel_capacity", "tier" => "t1").set(senders.t1.capacity() as f64);
gauge!("substrate_channel_capacity", "tier" => "t2").set(senders.t2.capacity() as f64);
gauge!("substrate_channel_capacity", "tier" => "t3").set(senders.t3.capacity() as f64);
if let Some(stats) = memory_stats::memory_stats() {
gauge!("substrate_rss_bytes").set(stats.physical_mem as f64);
}
// ---- sensor data aggregates (per type) ----
let mut by_type: HashMap<&'static str, Aggregate> = HashMap::new();
for (st, data) in &sensors_q {
by_type
.entry(st.0.label_str())
.or_insert_with(Aggregate::new)
.push(data.raw_value);
}
for (label, agg) in &by_type {
gauge!("sensor_aggregate", "type" => *label, "stat" => "count").set(agg.count as f64);
if agg.count > 0 {
gauge!("sensor_aggregate", "type" => *label, "stat" => "mean").set(agg.mean());
gauge!("sensor_aggregate", "type" => *label, "stat" => "min").set(agg.min);
gauge!("sensor_aggregate", "type" => *label, "stat" => "max").set(agg.max);
}
}
}
pub(super) fn diagnostics_system(
mut state: ResMut<DiagnosticsState>,
registry: Res<SensorRegistry>,
) {
state.ticks_since_log += 1;
let now = Instant::now();
let elapsed = now.duration_since(state.last_log);
if elapsed >= Duration::from_secs(1) {
let tick_hz = state.ticks_since_log as f64 / elapsed.as_secs_f64();
gauge!("substrate_tick_hz").set(tick_hz);
tracing::info!(
tick_hz = format_args!("{:.1}", tick_hz),
entities = registry.entity_count(),
"diagnostics"
);
state.last_log = now;
state.ticks_since_log = 0;
}
}
fn now_us() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_micros() as u64)
.unwrap_or(0)
}
/// Per-type accumulator for `export_system`'s sensor aggregates. NaN-safe.
#[derive(Debug, Clone, Copy)]
struct Aggregate {
count: u64,
sum: f64,
min: f64,
max: f64,
}
impl Aggregate {
fn new() -> Self {
Self {
count: 0,
sum: 0.0,
min: f64::INFINITY,
max: f64::NEG_INFINITY,
}
}
fn push(&mut self, v: f64) {
if !v.is_finite() {
return;
}
self.count += 1;
self.sum += v;
if v < self.min {
self.min = v;
}
if v > self.max {
self.max = v;
}
}
fn mean(&self) -> f64 {
if self.count == 0 {
f64::NAN
} else {
self.sum / self.count as f64
}
}
}

View File

@@ -0,0 +1,294 @@
//! Unit tests for the world's components and systems.
//!
//! Lives as a child module so it can poke at `pub(super)` items (the
//! internal resources, `threshold_for`, etc.) without enlarging the
//! public API.
use std::sync::Mutex;
use bevy::prelude::*;
use bevy::state::app::StatesPlugin;
use tokio::sync::{mpsc, oneshot};
use uuid::Uuid;
use crate::transport::ecs::{BridgeReceivers, BridgeSenders};
use crate::transport::state::ServerState;
use crate::transport::{QuicMessage, SensorType, T1Sender, T2Sender, T3Inbound, T3Sender};
use super::WorldPlugin;
use super::components::{RawSensorData, SMOOTHED_WINDOW, SmoothedValue, threshold_for};
use super::resources::SensorRegistry;
/// Build a Bevy app with just enough plugins/resources to run the world
/// systems against test-owned channels. No QUIC, no tokio runtime.
fn make_test_app() -> (
App,
mpsc::Sender<QuicMessage>,
mpsc::Sender<QuicMessage>,
mpsc::Sender<T3Inbound>,
) {
let (t1_tx, t1_rx) = mpsc::channel::<QuicMessage>(64);
let (t2_tx, t2_rx) = mpsc::channel::<QuicMessage>(64);
let (t3_tx, t3_rx) = mpsc::channel::<T3Inbound>(64);
let bridge = BridgeReceivers {
t1: Mutex::new(t1_rx),
t2: Mutex::new(t2_rx),
t3: Mutex::new(t3_rx),
};
// export_system samples channel depth/capacity from the senders; it
// requires the resource even when the test pushes via the raw senders
// directly (which is what the rest of the test does).
let senders = BridgeSenders {
t1: T1Sender::new(t1_tx.clone()),
t2: T2Sender::new(t2_tx.clone()),
t3: T3Sender::new(t3_tx.clone()),
};
let mut app = App::new();
app.add_plugins(MinimalPlugins)
.add_plugins(StatesPlugin)
.init_state::<ServerState>()
.insert_resource(bridge)
.insert_resource(senders)
.add_plugins(WorldPlugin);
// Force the state machine into Started so the run_if guard passes.
app.world_mut()
.resource_mut::<NextState<ServerState>>()
.set(ServerState::Started);
// Process the state transition before tests push messages.
app.update();
(app, t1_tx, t2_tx, t3_tx)
}
// ---- ingest_system: entity lifecycle and T3 ack semantics ----
#[test]
fn ingest_t1_creates_entity_and_writes_raw_data() {
let (mut app, t1_tx, _t2_tx, _t3_tx) = make_test_app();
let device = Uuid::from_u128(0xa1a2_a3a4_a5a6_a7a8_a9aa_abac_adae_afb0);
let msg = QuicMessage {
device_id: device,
sensor_id: 5,
raw_value: 3.14,
timestamp_us: 1_700_000_000_000_001,
sequence_number: 1,
sensor_type: SensorType::Temperature.as_u8(),
};
t1_tx.try_send(msg).expect("channel cap");
// Tick 1: ingest drains the channel and spawns via Commands.
app.update();
// Tick 2: Commands have flushed into the archetype.
app.update();
let registry = app.world().resource::<SensorRegistry>();
assert_eq!(registry.map.len(), 1);
let entity = *registry
.map
.get(&(device, 5))
.expect("entity not registered");
let data = app
.world()
.get::<RawSensorData>(entity)
.expect("RawSensorData missing");
assert_eq!(data.raw_value, 3.14);
assert_eq!(data.sequence_number, 1);
assert_eq!(data.timestamp_us, 1_700_000_000_000_001);
}
#[test]
fn ingest_t1_repeated_messages_update_in_place() {
let (mut app, t1_tx, _t2_tx, _t3_tx) = make_test_app();
let device = Uuid::new_v4();
// First reading.
t1_tx
.try_send(QuicMessage {
device_id: device,
sensor_id: 0,
raw_value: 1.0,
timestamp_us: 1,
sequence_number: 1,
sensor_type: SensorType::Generic.as_u8(),
})
.unwrap();
app.update();
app.update();
// Second reading on the same (device, sensor).
t1_tx
.try_send(QuicMessage {
device_id: device,
sensor_id: 0,
raw_value: 2.0,
timestamp_us: 2,
sequence_number: 2,
sensor_type: SensorType::Generic.as_u8(),
})
.unwrap();
app.update();
let registry = app.world().resource::<SensorRegistry>();
assert_eq!(registry.map.len(), 1, "should reuse the same entity");
let entity = *registry.map.get(&(device, 0)).unwrap();
let data = app.world().get::<RawSensorData>(entity).unwrap();
assert_eq!(data.raw_value, 2.0);
assert_eq!(data.sequence_number, 2);
}
#[test]
fn ingest_t3_replies_with_current_sensor_value() {
let (mut app, t1_tx, _t2_tx, t3_tx) = make_test_app();
let device = Uuid::new_v4();
// Seed a T1 reading so the (device, sensor) entity exists.
t1_tx
.try_send(QuicMessage {
device_id: device,
sensor_id: 9,
raw_value: 42.0,
timestamp_us: 1,
sequence_number: 1,
sensor_type: SensorType::Temperature.as_u8(),
})
.unwrap();
app.update();
app.update();
// Send a T3 command and capture the ack via the oneshot.
let (reply_tx, reply_rx) = oneshot::channel();
t3_tx
.try_send(T3Inbound {
command: QuicMessage {
device_id: device,
sensor_id: 9,
raw_value: 0.0,
timestamp_us: 0,
sequence_number: 7,
sensor_type: SensorType::Temperature.as_u8(),
},
reply: reply_tx,
})
.unwrap();
app.update();
let ack = reply_rx
.blocking_recv()
.expect("ECS handler should have replied");
assert_eq!(ack.device_id, device);
assert_eq!(ack.sensor_id, 9);
assert_eq!(ack.sequence_number, 7, "ack preserves correlation id");
assert_eq!(ack.raw_value, 42.0, "ack carries the latest sensor reading");
assert_eq!(
ack.typ(),
SensorType::Temperature,
"ack preserves sensor type"
);
assert!(ack.timestamp_us > 0, "ack stamped with server time");
}
// ---- SmoothedValue unit tests ----
#[test]
fn smoothed_value_first_push_sets_mean() {
let mut s = SmoothedValue::default();
s.push(10.0);
assert_eq!(s.mean, 10.0);
assert!(!s.above_threshold);
}
#[test]
fn smoothed_value_averages_filled_window() {
let mut s = SmoothedValue::default();
for v in [1.0, 2.0, 3.0, 4.0] {
s.push(v);
}
assert!((s.mean - 2.5).abs() < 1e-9);
}
#[test]
fn smoothed_value_rolls_after_window_fills() {
let mut s = SmoothedValue::default();
for _ in 0..SMOOTHED_WINDOW {
s.push(0.0);
}
assert!((s.mean - 0.0).abs() < 1e-9);
for _ in 0..SMOOTHED_WINDOW {
s.push(10.0);
}
assert!((s.mean - 10.0).abs() < 1e-9, "ring should fully roll over");
}
#[test]
fn smoothed_value_ignores_nonfinite() {
let mut s = SmoothedValue::default();
s.push(5.0);
let before = s.mean;
s.push(f64::NAN);
s.push(f64::INFINITY);
s.push(f64::NEG_INFINITY);
assert_eq!(s.mean, before, "non-finite values should not perturb the mean");
}
// ---- simulation_system: end-to-end threshold-crossing transition ----
#[test]
fn simulation_smoothes_and_detects_threshold_crossing() {
let (mut app, t1_tx, _t2_tx, _t3_tx) = make_test_app();
let device = Uuid::new_v4();
let threshold = threshold_for(SensorType::Temperature); // 22.0 °C
// Below-threshold readings: smoothed mean stays under, no crossing.
for seq in 0..SMOOTHED_WINDOW as u32 {
t1_tx
.try_send(QuicMessage {
device_id: device,
sensor_id: 0,
raw_value: 18.0,
timestamp_us: u64::from(seq),
sequence_number: seq,
sensor_type: SensorType::Temperature.as_u8(),
})
.unwrap();
app.update();
app.update();
}
let registry = app.world().resource::<SensorRegistry>();
let entity = *registry.map.get(&(device, 0)).unwrap();
let smoothed = app
.world()
.get::<SmoothedValue>(entity)
.expect("SmoothedValue should be on every sensor entity");
assert!(smoothed.mean < threshold);
assert!(!smoothed.above_threshold, "should not have crossed up yet");
// Above-threshold readings: enough samples to drag the mean above
// the threshold (window = 16; pushing 30°C for 16 ticks lands mean ≈ 30).
for seq in (SMOOTHED_WINDOW as u32)..(SMOOTHED_WINDOW as u32 * 2) {
t1_tx
.try_send(QuicMessage {
device_id: device,
sensor_id: 0,
raw_value: 30.0,
timestamp_us: u64::from(seq),
sequence_number: seq,
sensor_type: SensorType::Temperature.as_u8(),
})
.unwrap();
app.update();
}
let smoothed = app.world().get::<SmoothedValue>(entity).unwrap();
assert!(smoothed.mean > threshold);
assert!(
smoothed.above_threshold,
"smoothed mean should have crossed up through {threshold}"
);
}