diff --git a/agent/protector-agent/src/coalesce/tests.rs b/agent/protector-agent/src/coalesce/tests.rs index f6ec4f1..458db0e 100644 --- a/agent/protector-agent/src/coalesce/tests.rs +++ b/agent/protector-agent/src/coalesce/tests.rs @@ -9,6 +9,7 @@ fn obs(pod_uid: &str, behavior: Behavior, at_ms: u64) -> RuntimeObservation { attribution: Attribution::by_pod_uid(pod_uid), source: Some("protector-agent".into()), observed_at_ms: Some(at_ms), + node: None, behavior, } } diff --git a/agent/protector-agent/src/main.rs b/agent/protector-agent/src/main.rs index bb1db72..611761e 100644 --- a/agent/protector-agent/src/main.rs +++ b/agent/protector-agent/src/main.rs @@ -13,14 +13,62 @@ mod pod; mod reporter; use std::io::IsTerminal; -use std::time::Duration; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use protector_behavior::RuntimeObservation; +use protector_behavior::{AgentReport, RuntimeObservation}; use tokio::sync::mpsc; use coalesce::Coalescer; use reporter::Reporter; +/// Shared **probe-attach status** (JEF-308): the observer sets it once its eBPF probes attach, and +/// the per-node liveness beacon reads it each window. The default no-eBPF build never sets it (stays +/// `0/0`), so the agent honestly reports itself BLIND — signal-flow liveness, not pod-Ready. +#[derive(Default)] +pub struct ProbeStatus { + loaded: AtomicU32, + total: AtomicU32, +} + +impl ProbeStatus { + /// Record how many probes attached out of how many were tried (called by the observer). + pub fn set(&self, loaded: u32, total: u32) { + self.loaded.store(loaded, Ordering::Relaxed); + self.total.store(total, Ordering::Relaxed); + } + + /// `(loaded, total)` as of now — read by the liveness beacon. + pub fn snapshot(&self) -> (u32, u32) { + ( + self.loaded.load(Ordering::Relaxed), + self.total.load(Ordering::Relaxed), + ) + } +} + +/// Wall-clock now as Unix epoch millis, for the beacon's freshness stamp. `None` only if the clock +/// is before the epoch — the engine then stamps ingest time. +fn now_ms() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .map(|d| d.as_millis() as u64) +} + +/// Build a per-node liveness beacon (JEF-308) from the node, the probe-attach status, and the +/// signals emitted this window. Pure over its inputs so it's unit-testable without the runtime. +fn build_agent_report(node: &str, probes: (u32, u32), signals: u64) -> AgentReport { + AgentReport { + node: node.to_string(), + probes_loaded: probes.0, + probes_total: probes.1, + signals_emitted: signals, + observed_at_ms: now_ms(), + } +} + /// Max distinct coalesced keys the debounce buffer holds before a forced flush (JEF-296). /// Bounds memory and keeps a flushed batch well under the engine's 1024 per-batch cap, so /// the "behavior batch exceeds the per-batch cap" WARN stays quiet under normal load. @@ -63,12 +111,26 @@ async fn main() -> anyhow::Result<()> { let endpoint = std::env::var("PROTECTOR_AGENT_ENDPOINT") .unwrap_or_else(|_| "http://protector.protector.svc.cluster.local:9999".to_string()); let debounce_window = parse_debounce_window(std::env::var("PROTECTOR_AGENT_DEBOUNCE_MS").ok()); + // The node this agent runs on (JEF-308), from the downward API (`K8S_NODE = spec.nodeName`). + // Stamped onto every observation and onto the per-node liveness beacon. When unset (a dev run + // outside k8s) we can't attribute per node — observations go out node-less and no beacon is + // sent (an un-attributable beacon would be dishonest). + let node = std::env::var("K8S_NODE").ok().filter(|n| !n.is_empty()); + if node.is_none() { + tracing::warn!( + "K8S_NODE is unset — observations are not node-attributed and no per-node liveness \ + beacon will be sent (set it via the downward API `spec.nodeName`)" + ); + } tracing::info!( %endpoint, + node = node.as_deref().unwrap_or(""), debounce_ms = debounce_window.as_millis(), "protector-agent starting" ); let mut reporter = Reporter::new(&endpoint)?; + // Probe-attach status the observer updates and the liveness beacon reads (JEF-308). + let probes = Arc::new(ProbeStatus::default()); let (tx, mut rx) = mpsc::channel::(4096); @@ -79,6 +141,8 @@ async fn main() -> anyhow::Result<()> { // corroboration must stay low-latency). Best-effort sends — a lost batch costs freshness, // never correctness (behavioral evidence is additive). A running count is logged at info // once per HEARTBEAT_INTERVAL so an operator can confirm the agent is actually reporting. + let beacon_node = node.clone(); + let beacon_probes = probes.clone(); let flusher = tokio::spawn(async move { let mut coalescer = Coalescer::new(MAX_BATCH); let mut reported_since_tick: usize = 0; @@ -91,7 +155,9 @@ async fn main() -> anyhow::Result<()> { loop { tokio::select! { recv = rx.recv() => match recv { - Some(obs) => { + Some(mut obs) => { + // Stamp this agent's node (JEF-308) so the observation is node-attributed. + obs.node = beacon_node.clone(); // `offer` returns anything to POST NOW: an alert (never debounced), // or the drained buffer if this new distinct key hit the max-size cap. let immediate = coalescer.offer(obs); @@ -124,20 +190,35 @@ async fn main() -> anyhow::Result<()> { "behavioral observations reported (last {}s)", HEARTBEAT_INTERVAL.as_secs(), ); + // Per-node liveness beacon (JEF-308): sent EVERY window even when quiet + // (reported_since_tick == 0) — a quiet node with probes loaded reads + // healthy-quiet, not blind. Skipped only when the node is unknown (an + // un-attributable beacon would be dishonest). probes==0/0 ⇒ blind (Ready + // but no probe attached, or the no-eBPF build). + if let Some(node) = &beacon_node { + let report = build_agent_report( + node, + beacon_probes.snapshot(), + reported_since_tick as u64, + ); + reporter.send_report(&report).await; + } reported_since_tick = 0; } } } }); - // Collection. Default build is a no-op; `--features ebpf` loads the real probes. + // Collection. Default build is a no-op; `--features ebpf` loads the real probes. The observer + // updates `probes` with how many eBPF probes attached (JEF-308) — the no-op build leaves it at + // 0/0, honestly reporting itself blind. #[cfg(not(feature = "ebpf"))] - observer::NoopObserver.run(tx).await; + observer::NoopObserver.run(tx, probes).await; #[cfg(feature = "ebpf")] { // Events are attributed by pod UID (from the cgroup); the engine resolves UID → // namespace/pod via its watch, so the agent needs no cluster credentials. - if let Err(error) = observer::EbpfObserver.run(tx).await { + if let Err(error) = observer::EbpfObserver.run(tx, probes).await { // Degrade, don't crashloop (ADR-0014): a missing hook / failed attach should // leave the pod up for inspection, not hammer restarts. tracing::error!(%error, "ebpf observer exited; idling (no collection)"); @@ -166,6 +247,32 @@ mod tests { ); } + #[test] + fn agent_report_carries_node_probes_and_window_signals() { + // JEF-308: a healthy window — probes loaded, some signals. + let r = build_agent_report("node-a", (6, 6), 12); + assert_eq!(r.node, "node-a"); + assert_eq!(r.probes_loaded, 6); + assert_eq!(r.probes_total, 6); + assert_eq!(r.signals_emitted, 12); + assert!(!r.is_blind()); + assert!(!r.is_partial()); + } + + #[test] + fn agent_report_is_blind_when_no_probe_attached() { + // The no-eBPF build (or a Ready-but-blind agent) reports 0 probes → blind, even quiet. + let r = build_agent_report("node-a", (0, 0), 0); + assert!(r.is_blind()); + // A quiet window with probes loaded is NOT blind (quiet≠blind). + let quiet = build_agent_report("node-a", (6, 6), 0); + assert!(!quiet.is_blind()); + // A partial load is degraded, not blind. + let partial = build_agent_report("node-a", (4, 6), 1); + assert!(partial.is_partial()); + assert!(!partial.is_blind()); + } + #[test] fn debounce_window_falls_back_to_the_default() { // Unset, unparseable, and zero all fall back — a zero period would panic the diff --git a/agent/protector-agent/src/observer.rs b/agent/protector-agent/src/observer.rs index d56f9d2..fe3a7eb 100644 --- a/agent/protector-agent/src/observer.rs +++ b/agent/protector-agent/src/observer.rs @@ -19,7 +19,14 @@ pub struct NoopObserver; #[cfg(not(feature = "ebpf"))] impl NoopObserver { - pub async fn run(self, _tx: Sender) { + pub async fn run( + self, + _tx: Sender, + probes: std::sync::Arc, + ) { + // No collection ⇒ zero probes attached: the liveness beacon (JEF-308) then honestly reports + // this node BLIND (probes_loaded == 0), never a false healthy. + probes.set(0, 0); tracing::warn!( "built without the `ebpf` feature — no behavioral collection. Rebuild with \ `--features ebpf` on a node (needs bpf-linker) to load the probes." @@ -210,12 +217,17 @@ mod ebpf { pub struct EbpfObserver; impl EbpfObserver { - pub async fn run(self, tx: Sender) -> anyhow::Result<()> { + pub async fn run( + self, + tx: Sender, + probes: Arc, + ) -> anyhow::Result<()> { // The BPF object is compiled + embedded by build.rs under the `ebpf` feature. let mut ebpf = Ebpf::load(aya::include_bytes_aligned!(concat!( env!("OUT_DIR"), "/protector-agent.bpf.o" )))?; + let mut loaded: u32 = 0; for (name, hook) in PROBES { let program: &mut KProbe = ebpf .program_mut(name) @@ -223,12 +235,20 @@ mod ebpf { .try_into()?; program.load()?; program.attach(*hook, 0)?; + loaded += 1; tracing::info!(probe = *name, hook = *hook, "attached probe"); } // Best-effort fentry probes (secret-read, library-load). NOT fatal — a probe // that fails to load/attach (no BTF/fentry, verifier reject) is logged and // skipped, leaving the others (and the connect kprobe) running. - Self::attach_fentry(&mut ebpf); + let (fentry_loaded, fentry_total) = Self::attach_fentry(&mut ebpf); + loaded += fentry_loaded; + // Publish probe-attach status (JEF-308): the liveness beacon reads it so a Ready agent + // whose probes failed to attach (loaded == 0) reads BLIND, and a partial load reads + // degraded — signal-flow liveness, not pod-Ready. + let total = PROBES.len() as u32 + fentry_total; + probes.set(loaded, total); + tracing::info!(loaded, total, "eBPF probes attached (JEF-308 liveness)"); tracing::info!("draining events"); let ring = RingBuf::try_from( @@ -453,6 +473,9 @@ mod ebpf { attribution: Attribution::by_pod_uid(uid), source: Some(SOURCE.into()), observed_at_ms: now_ms(), + // The agent's node (JEF-308) is stamped by the flusher in `main` from `K8S_NODE` + // — kept in one place, so the ebpf worker leaves it unset here. + node: None, behavior: raw.into_behavior(), }; if tx.send(obs).await.is_err() { @@ -505,8 +528,10 @@ mod ebpf { /// Attach the fentry probes, each best-effort (a failure is logged, not fatal). /// (program name in the object, kernel function it hooks). fentry attaches via - /// BTF, so it's separate from the kprobe table; the BTF is loaded once. - fn attach_fentry(ebpf: &mut Ebpf) { + /// BTF, so it's separate from the kprobe table; the BTF is loaded once. Returns + /// `(attached, attempted)` so the caller can publish the probe-attach status the + /// per-node liveness beacon reads (JEF-308) — a partial load reads degraded. + fn attach_fentry(ebpf: &mut Ebpf) -> (u32, u32) { const FENTRY_PROBES: &[(&str, &str)] = &[ ("file_open", "security_file_open"), ("file_write", "security_file_open"), @@ -514,21 +539,27 @@ mod ebpf { ("fix_setuid", "security_task_fix_setuid"), ("bprm_check", "security_bprm_check"), ]; + let attempted = FENTRY_PROBES.len() as u32; let btf = match Btf::from_sys_fs() { Ok(btf) => btf, Err(error) => { tracing::warn!(%error, "kernel BTF unavailable; fentry probes off"); - return; + return (0, attempted); } }; + let mut attached = 0u32; for (name, func) in FENTRY_PROBES { match Self::attach_one_fentry(ebpf, &btf, name, func) { - Ok(()) => tracing::info!(probe = *name, func = *func, "attached fentry"), + Ok(()) => { + attached += 1; + tracing::info!(probe = *name, func = *func, "attached fentry"); + } Err(error) => { tracing::warn!(%error, probe = *name, "fentry did not attach; continuing") } } } + (attached, attempted) } fn attach_one_fentry( @@ -724,233 +755,8 @@ mod ebpf { } #[cfg(test)] - mod tests { - use super::*; - - const POD_CGROUP: &str = - "/kubepods/besteffort/pod3f5e1a2b-4c6d-7e8f-9a0b-1c2d3e4f5a6b/abc123"; - const POD_SLICE: &str = "/sys/fs/cgroup/kubepods.slice/\ - kubepods-besteffort-pod3f5e1a2b_4c6d_7e8f_9a0b_1c2d3e4f5a6b.slice"; - const POD_UID: &str = "3f5e1a2b-4c6d-7e8f-9a0b-1c2d3e4f5a6b"; - - fn attr(pid: u32, cgroup_id: u64) -> EventAttr { - EventAttr { pid, cgroup_id } - } - - #[test] - fn resolve_uses_the_table_and_never_reads_proc_on_a_cgroup_id_hit() { - // JEF-158 hot path: a cgroup_id table hit resolves with NO `/proc` read and NO - // fallback-cache entry — which is what lets an already-exited process attribute. - let table = crate::pod::build_cgroup_table([(100u64, POD_SLICE.to_string())]); - let mut cache = HashMap::new(); - assert_eq!( - EbpfObserver::resolve(&table, &mut cache, attr(4321, 100)), - PodAttribution::Pod(POD_UID.to_string()) - ); - assert!( - cache.is_empty(), - "a table hit must not touch the /proc cache" - ); - } - - #[test] - fn resolve_falls_back_to_proc_and_memoizes_on_a_table_miss() { - // A table miss falls through to the `/proc` read; a flood from one pid must not - // re-read it (the fallback cache). - let table = CgroupTable::default(); - // Seed the fallback cache as the worker would, then confirm a repeat is cached. - let mut cache = HashMap::new(); - let resolved = EbpfObserver::resolve(&table, &mut cache, attr(7, 0)); - // pid 7 / cgroup_id 0 → table miss → /proc read of /proc/7/cgroup (absent in the - // test env) → Unreadable, and it's cached. - assert_eq!(resolved, PodAttribution::Unreadable); - assert!(cache.contains_key(&7), "fallback result should be cached"); - } - - #[test] - fn resolve_clears_fallback_cache_at_cap() { - let table = CgroupTable::default(); - let mut cache = HashMap::new(); - // Fill the fallback cache to capacity (each distinct miss pid inserts once). - for pid in 0..PID_CACHE_CAP as u32 { - EbpfObserver::resolve(&table, &mut cache, attr(pid, 0)); - } - assert_eq!(cache.len(), PID_CACHE_CAP); - EbpfObserver::resolve(&table, &mut cache, attr(PID_CACHE_CAP as u32, 0)); - assert_eq!(cache.len(), 1, "cache should clear wholesale at the cap"); - } - - #[test] - fn proc_fallback_classifies_pod_cgroup_text() { - // The injected `/proc` reader path (via pod::resolve_attribution) still maps a - // pod cgroup text to its UID — the fallback for a table miss is unchanged. - assert_eq!( - crate::pod::resolve_attribution(&CgroupTable::default(), 0, 42, |_| Some( - POD_CGROUP.to_string() - )), - PodAttribution::Pod(POD_UID.to_string()) - ); - } - - #[test] - fn decode_connect_parses_without_proc() { - let ev = ConnEvent { - header: EventHeader { - kind: KIND_CONNECT, - pid: 1234, - cgroup_id: 777, - }, - daddr: u32::from_ne_bytes([8, 8, 8, 8]), - dport: 443, - }; - let bytes = unsafe { - std::slice::from_raw_parts( - (&ev as *const ConnEvent).cast::(), - std::mem::size_of::(), - ) - }; - match EbpfObserver::decode(bytes) { - Some(RawEvent::Connect { attr, daddr, dport }) => { - assert_eq!(attr.pid, 1234); - assert_eq!(attr.cgroup_id, 777); - assert_eq!(daddr, ev.daddr); - assert_eq!(dport, 443); - } - other => panic!("expected Connect, got something else: {}", other.is_none()), - } - } - - #[test] - fn decode_priv_change_parses_uids() { - let ev = PrivEvent { - header: EventHeader { - kind: KIND_PRIV_CHANGE, - pid: 4321, - cgroup_id: 888, - }, - old_uid: 1000, - new_uid: 0, - }; - let bytes = unsafe { - std::slice::from_raw_parts( - (&ev as *const PrivEvent).cast::(), - std::mem::size_of::(), - ) - }; - match EbpfObserver::decode(bytes) { - Some(RawEvent::PrivChange { - attr, - old_uid, - new_uid, - }) => { - assert_eq!(attr.pid, 4321); - assert_eq!(attr.cgroup_id, 888); - assert_eq!(old_uid, 1000); - assert_eq!(new_uid, 0); - } - _ => panic!("expected PrivChange"), - } - // And the raw event maps to the PrivilegeChange behavior with from/to uids. - let raw = EbpfObserver::decode(bytes).unwrap(); - assert_eq!( - raw.into_behavior(), - Behavior::PrivilegeChange { - from_uid: 1000, - to_uid: 0, - } - ); - } - - #[test] - fn decode_exec_parses_path_and_maps_to_process_exec() { - // A KIND_EXEC FileEvent carrying a NUL-terminated exec path must decode to a - // RawEvent::Exec, and into_behavior must map it to Behavior::ProcessExec whose - // fingerprint coarsens to the basename (JEF-53). - let mut path = [0u8; PATH_CAP]; - let bin = b"/usr/bin/bash\0"; - path[..bin.len()].copy_from_slice(bin); - let ev = FileEvent { - header: EventHeader { - kind: KIND_EXEC, - pid: 4321, - cgroup_id: 999, - }, - len: bin.len() as u32, - path, - }; - let bytes = unsafe { - std::slice::from_raw_parts( - (&ev as *const FileEvent).cast::(), - std::mem::size_of::(), - ) - }; - let raw = EbpfObserver::decode(bytes).expect("KIND_EXEC should decode"); - match &raw { - RawEvent::Exec { attr, path } => { - assert_eq!(attr.pid, 4321); - assert_eq!(attr.cgroup_id, 999); - assert_eq!(path, "/usr/bin/bash"); - } - _ => panic!("expected RawEvent::Exec"), - } - assert_eq!(raw.attr().pid, 4321); - match raw.into_behavior() { - Behavior::ProcessExec { path } => { - assert_eq!(path, "/usr/bin/bash"); - assert_eq!( - Behavior::ProcessExec { path }.fingerprint_key(), - "exec:bash" - ); - } - other => panic!("expected ProcessExec, got {other:?}"), - } - } - - #[test] - fn decode_file_write_parses_path_and_maps_to_file_write() { - // A KIND_FILE_WRITE FileEvent carrying a NUL-terminated path must decode to a - // RawEvent::FileWrite with attribution, and into_behavior must map it to - // Behavior::FileWrite whose fingerprint coarsens to the dirname (JEF-306). - let mut path = [0u8; PATH_CAP]; - let file = b"/etc/cron.d/dropper\0"; - path[..file.len()].copy_from_slice(file); - let ev = FileEvent { - header: EventHeader { - kind: KIND_FILE_WRITE, - pid: 555, - cgroup_id: 4242, - }, - len: file.len() as u32, - path, - }; - let bytes = unsafe { - std::slice::from_raw_parts( - (&ev as *const FileEvent).cast::(), - std::mem::size_of::(), - ) - }; - let raw = EbpfObserver::decode(bytes).expect("KIND_FILE_WRITE should decode"); - match &raw { - RawEvent::FileWrite { attr, path } => { - assert_eq!(attr.pid, 555); - assert_eq!(attr.cgroup_id, 4242); - assert_eq!(path, "/etc/cron.d/dropper"); - } - _ => panic!("expected RawEvent::FileWrite"), - } - assert_eq!(raw.attr().cgroup_id, 4242); - match raw.into_behavior() { - Behavior::FileWrite { path } => { - assert_eq!(path, "/etc/cron.d/dropper"); - assert_eq!( - Behavior::FileWrite { path }.fingerprint_key(), - "write:/etc/cron.d" - ); - } - other => panic!("expected FileWrite, got {other:?}"), - } - } - } + #[path = "observer_ebpf_tests.rs"] + mod tests; } #[cfg(test)] diff --git a/agent/protector-agent/src/observer/ebpf/observer_ebpf_tests.rs b/agent/protector-agent/src/observer/ebpf/observer_ebpf_tests.rs new file mode 100644 index 0000000..81f9f3a --- /dev/null +++ b/agent/protector-agent/src/observer/ebpf/observer_ebpf_tests.rs @@ -0,0 +1,228 @@ +//! eBPF-observer unit tests (decode / attribution), extracted from `observer.rs` to keep it +//! under the 1,000-line cap (CLAUDE.md). Included via `#[path]` inside `mod ebpf`, so `super` +//! resolves to the eBPF observer module. + +use super::*; + +const POD_CGROUP: &str = "/kubepods/besteffort/pod3f5e1a2b-4c6d-7e8f-9a0b-1c2d3e4f5a6b/abc123"; +const POD_SLICE: &str = "/sys/fs/cgroup/kubepods.slice/\ + kubepods-besteffort-pod3f5e1a2b_4c6d_7e8f_9a0b_1c2d3e4f5a6b.slice"; +const POD_UID: &str = "3f5e1a2b-4c6d-7e8f-9a0b-1c2d3e4f5a6b"; + +fn attr(pid: u32, cgroup_id: u64) -> EventAttr { + EventAttr { pid, cgroup_id } +} + +#[test] +fn resolve_uses_the_table_and_never_reads_proc_on_a_cgroup_id_hit() { + // JEF-158 hot path: a cgroup_id table hit resolves with NO `/proc` read and NO + // fallback-cache entry — which is what lets an already-exited process attribute. + let table = crate::pod::build_cgroup_table([(100u64, POD_SLICE.to_string())]); + let mut cache = HashMap::new(); + assert_eq!( + EbpfObserver::resolve(&table, &mut cache, attr(4321, 100)), + PodAttribution::Pod(POD_UID.to_string()) + ); + assert!( + cache.is_empty(), + "a table hit must not touch the /proc cache" + ); +} + +#[test] +fn resolve_falls_back_to_proc_and_memoizes_on_a_table_miss() { + // A table miss falls through to the `/proc` read; a flood from one pid must not + // re-read it (the fallback cache). + let table = CgroupTable::default(); + // Seed the fallback cache as the worker would, then confirm a repeat is cached. + let mut cache = HashMap::new(); + let resolved = EbpfObserver::resolve(&table, &mut cache, attr(7, 0)); + // pid 7 / cgroup_id 0 → table miss → /proc read of /proc/7/cgroup (absent in the + // test env) → Unreadable, and it's cached. + assert_eq!(resolved, PodAttribution::Unreadable); + assert!(cache.contains_key(&7), "fallback result should be cached"); +} + +#[test] +fn resolve_clears_fallback_cache_at_cap() { + let table = CgroupTable::default(); + let mut cache = HashMap::new(); + // Fill the fallback cache to capacity (each distinct miss pid inserts once). + for pid in 0..PID_CACHE_CAP as u32 { + EbpfObserver::resolve(&table, &mut cache, attr(pid, 0)); + } + assert_eq!(cache.len(), PID_CACHE_CAP); + EbpfObserver::resolve(&table, &mut cache, attr(PID_CACHE_CAP as u32, 0)); + assert_eq!(cache.len(), 1, "cache should clear wholesale at the cap"); +} + +#[test] +fn proc_fallback_classifies_pod_cgroup_text() { + // The injected `/proc` reader path (via pod::resolve_attribution) still maps a + // pod cgroup text to its UID — the fallback for a table miss is unchanged. + assert_eq!( + crate::pod::resolve_attribution(&CgroupTable::default(), 0, 42, |_| Some( + POD_CGROUP.to_string() + )), + PodAttribution::Pod(POD_UID.to_string()) + ); +} + +#[test] +fn decode_connect_parses_without_proc() { + let ev = ConnEvent { + header: EventHeader { + kind: KIND_CONNECT, + pid: 1234, + cgroup_id: 777, + }, + daddr: u32::from_ne_bytes([8, 8, 8, 8]), + dport: 443, + }; + let bytes = unsafe { + std::slice::from_raw_parts( + (&ev as *const ConnEvent).cast::(), + std::mem::size_of::(), + ) + }; + match EbpfObserver::decode(bytes) { + Some(RawEvent::Connect { attr, daddr, dport }) => { + assert_eq!(attr.pid, 1234); + assert_eq!(attr.cgroup_id, 777); + assert_eq!(daddr, ev.daddr); + assert_eq!(dport, 443); + } + other => panic!("expected Connect, got something else: {}", other.is_none()), + } +} + +#[test] +fn decode_priv_change_parses_uids() { + let ev = PrivEvent { + header: EventHeader { + kind: KIND_PRIV_CHANGE, + pid: 4321, + cgroup_id: 888, + }, + old_uid: 1000, + new_uid: 0, + }; + let bytes = unsafe { + std::slice::from_raw_parts( + (&ev as *const PrivEvent).cast::(), + std::mem::size_of::(), + ) + }; + match EbpfObserver::decode(bytes) { + Some(RawEvent::PrivChange { + attr, + old_uid, + new_uid, + }) => { + assert_eq!(attr.pid, 4321); + assert_eq!(attr.cgroup_id, 888); + assert_eq!(old_uid, 1000); + assert_eq!(new_uid, 0); + } + _ => panic!("expected PrivChange"), + } + // And the raw event maps to the PrivilegeChange behavior with from/to uids. + let raw = EbpfObserver::decode(bytes).unwrap(); + assert_eq!( + raw.into_behavior(), + Behavior::PrivilegeChange { + from_uid: 1000, + to_uid: 0, + } + ); +} + +#[test] +fn decode_exec_parses_path_and_maps_to_process_exec() { + // A KIND_EXEC FileEvent carrying a NUL-terminated exec path must decode to a + // RawEvent::Exec, and into_behavior must map it to Behavior::ProcessExec whose + // fingerprint coarsens to the basename (JEF-53). + let mut path = [0u8; PATH_CAP]; + let bin = b"/usr/bin/bash\0"; + path[..bin.len()].copy_from_slice(bin); + let ev = FileEvent { + header: EventHeader { + kind: KIND_EXEC, + pid: 4321, + cgroup_id: 999, + }, + len: bin.len() as u32, + path, + }; + let bytes = unsafe { + std::slice::from_raw_parts( + (&ev as *const FileEvent).cast::(), + std::mem::size_of::(), + ) + }; + let raw = EbpfObserver::decode(bytes).expect("KIND_EXEC should decode"); + match &raw { + RawEvent::Exec { attr, path } => { + assert_eq!(attr.pid, 4321); + assert_eq!(attr.cgroup_id, 999); + assert_eq!(path, "/usr/bin/bash"); + } + _ => panic!("expected RawEvent::Exec"), + } + assert_eq!(raw.attr().pid, 4321); + match raw.into_behavior() { + Behavior::ProcessExec { path } => { + assert_eq!(path, "/usr/bin/bash"); + assert_eq!( + Behavior::ProcessExec { path }.fingerprint_key(), + "exec:bash" + ); + } + other => panic!("expected ProcessExec, got {other:?}"), + } +} + +#[test] +fn decode_file_write_parses_path_and_maps_to_file_write() { + // A KIND_FILE_WRITE FileEvent carrying a NUL-terminated path must decode to a + // RawEvent::FileWrite with attribution, and into_behavior must map it to + // Behavior::FileWrite whose fingerprint coarsens to the dirname (JEF-306). + let mut path = [0u8; PATH_CAP]; + let file = b"/etc/cron.d/dropper\0"; + path[..file.len()].copy_from_slice(file); + let ev = FileEvent { + header: EventHeader { + kind: KIND_FILE_WRITE, + pid: 555, + cgroup_id: 4242, + }, + len: file.len() as u32, + path, + }; + let bytes = unsafe { + std::slice::from_raw_parts( + (&ev as *const FileEvent).cast::(), + std::mem::size_of::(), + ) + }; + let raw = EbpfObserver::decode(bytes).expect("KIND_FILE_WRITE should decode"); + match &raw { + RawEvent::FileWrite { attr, path } => { + assert_eq!(attr.pid, 555); + assert_eq!(attr.cgroup_id, 4242); + assert_eq!(path, "/etc/cron.d/dropper"); + } + _ => panic!("expected RawEvent::FileWrite"), + } + assert_eq!(raw.attr().cgroup_id, 4242); + match raw.into_behavior() { + Behavior::FileWrite { path } => { + assert_eq!(path, "/etc/cron.d/dropper"); + assert_eq!( + Behavior::FileWrite { path }.fingerprint_key(), + "write:/etc/cron.d" + ); + } + other => panic!("expected FileWrite, got {other:?}"), + } +} diff --git a/agent/protector-agent/src/reporter.rs b/agent/protector-agent/src/reporter.rs index 32d2694..143f74f 100644 --- a/agent/protector-agent/src/reporter.rs +++ b/agent/protector-agent/src/reporter.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use protector_behavior::RuntimeObservation; +use protector_behavior::{AgentReport, RuntimeObservation}; /// Re-resolve the token after this many consecutive 401s. Small so a genuine skew heals /// fast, but >1 so a single transient 401 (e.g. an engine mid-restart that hasn't loaded @@ -39,10 +39,13 @@ const ERROR_EVERY_N_REJECTIONS: u64 = 20; /// (a stale-then-fresh token) without touching the filesystem or sleeping. type TokenSource = Box Option + Send>; -/// POSTs batches of [`RuntimeObservation`]s to `{base}/behavior`. +/// POSTs batches of [`RuntimeObservation`]s to `{base}/behavior`, and per-node liveness beacons +/// (JEF-308) to `{base}/agent-liveness`. pub struct Reporter { client: reqwest::Client, url: String, + /// The per-node agent-liveness beacon endpoint (`{base}/agent-liveness`, JEF-308). + liveness_url: String, /// Shared-secret bearer for the engine's ingest authn (Fix A). `None` = send no /// `Authorization` header (the engine then runs the ingest unauthenticated, which /// it warns about); set it once the Secret has rolled out. @@ -115,9 +118,11 @@ impl Reporter { without an Authorization header" ); } + let base = base.trim_end_matches('/'); Self { client, - url: format!("{}/behavior", base.trim_end_matches('/')), + url: format!("{base}/behavior"), + liveness_url: format!("{base}/agent-liveness"), token, token_source: source, consecutive_401s: 0, @@ -136,6 +141,36 @@ impl Reporter { req } + /// Build the POST for a per-node liveness beacon (JEF-308), attaching the same bearer as the + /// observation path. Split out so the header/URL wiring is unit-testable without a server. + fn build_report_request(&self, report: &AgentReport) -> reqwest::RequestBuilder { + let mut req = self.client.post(&self.liveness_url).json(report); + if let Some(token) = &self.token { + req = req.bearer_auth(token); + } + req + } + + /// POST one per-node liveness beacon (JEF-308) to `{base}/agent-liveness`. Best-effort like + /// [`send`](Self::send): a failed beacon is logged and dropped — a missed beacon costs a little + /// freshness, and the engine's TTL means a node that genuinely stops beaconing reads blind, + /// which is the honest outcome. Sent every window even when the node is quiet, so a quiet node + /// reads healthy-quiet, not blind. `&mut self` (like [`send`](Self::send)) so the future stays + /// `Send` for `tokio::spawn` — the boxed token source makes a shared `&Reporter` non-`Send`. + pub async fn send_report(&mut self, report: &AgentReport) { + match self.build_report_request(report).send().await { + Ok(resp) if resp.status().is_success() => { + tracing::debug!(node = %report.node, "reported agent liveness"); + } + Ok(resp) => { + tracing::warn!(status = %resp.status(), "agent-liveness ingest rejected beacon"); + } + Err(error) => { + tracing::warn!(%error, "agent-liveness ingest unreachable"); + } + } + } + /// Cumulative (delivered, rejected) tallies for the periodic heartbeat (JEF-240). /// `delivered` counts accepted observations; `rejected` counts rejected batches. pub fn counters(&self) -> (u64, u64) { @@ -241,6 +276,7 @@ mod tests { Reporter { client: reqwest::Client::new(), url: "http://engine.svc:9999/behavior".to_string(), + liveness_url: "http://engine.svc:9999/agent-liveness".to_string(), token: owned, token_source: Box::new(move || for_source.clone()), consecutive_401s: 0, @@ -254,6 +290,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "web"), source: Some("agent".into()), observed_at_ms: None, + node: None, behavior: Behavior::SecretRead { secret: "app/session-key".into(), source: SecretReadSource::Mounted, @@ -276,6 +313,28 @@ mod tests { assert_eq!(auth, "Bearer s3cr3t"); } + /// JEF-308: a liveness beacon POSTs to `{base}/agent-liveness` carrying the same bearer. + #[test] + fn liveness_beacon_targets_agent_liveness_with_bearer() { + let reporter = reporter_with(Some("s3cr3t")); + let report = AgentReport { + node: "node-a".into(), + probes_loaded: 6, + probes_total: 6, + signals_emitted: 3, + observed_at_ms: None, + }; + let req = reporter + .build_report_request(&report) + .build() + .expect("request builds"); + assert_eq!(req.url().as_str(), "http://engine.svc:9999/agent-liveness"); + assert_eq!( + req.headers().get(reqwest::header::AUTHORIZATION).unwrap(), + "Bearer s3cr3t" + ); + } + /// Without a token, no Authorization header is sent (the engine warns + accepts, /// or rejects if it has a token — the rollout-ordering contract). #[test] diff --git a/behavior/src/lib.rs b/behavior/src/lib.rs index 7358565..9897da1 100644 --- a/behavior/src/lib.rs +++ b/behavior/src/lib.rs @@ -301,10 +301,63 @@ pub struct RuntimeObservation { /// batch interval + a judging pass). Defaulted → adapter uses now(). #[serde(default, skip_serializing_if = "Option::is_none")] pub observed_at_ms: Option, + /// The Kubernetes NODE the sensor observed this on (JEF-308) — the eBPF agent reports its + /// own node (from the downward API, `spec.nodeName`), so the engine can reason about + /// runtime-corroboration coverage PER NODE ("blind on node X"), not just fleet-aggregate. + /// Defaulted (older agents / Falco, which is node-agnostic, omit it) — an absent node is + /// honestly node-unattributed, never guessed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub node: Option, /// What the workload actually did. pub behavior: Behavior, } +/// A per-node **agent-liveness beacon** (JEF-308): the eBPF agent's own self-report, one per +/// report window, distinct from a workload [`RuntimeObservation`]. It is what makes +/// runtime-corroboration coverage honestly derivable per node: liveness is **signal-flow**, not +/// pod-Ready — a Ready agent whose eBPF probes failed to attach is still BLIND (the exact Falco +/// failure mode), so it reports `probes_loaded = 0`, and the engine reads it as blind despite the +/// pod being up. +/// +/// Critically, the agent emits this **every window even when it saw nothing**, so a quiet node +/// (`signals_emitted = 0`, probes loaded) reads HEALTHY-quiet — NOT blind. Only a node that never +/// reports, or reports `probes_loaded = 0`, reads blind. Sent over the same in-cluster ingest the +/// observations already use (zero egress) — never the agent's OTLP/metrics endpoint. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentReport { + /// The node this agent runs on (downward API `spec.nodeName`). Untrusted-adjacent at + /// render — the engine escapes it like any cluster name (never `PreEscaped`). + pub node: String, + /// How many eBPF probes ACTUALLY attached this window. `0` ⇒ the agent is Ready but blind + /// (nothing is being observed); `< probes_total` ⇒ partial coverage (degraded). + pub probes_loaded: u32, + /// How many probes the agent tried to load — the denominator for "partial". `0` only for a + /// build with no collection (the default no-eBPF image), which is also honestly blind. + pub probes_total: u32, + /// Signals the agent emitted this window. `0` is HEALTHY-quiet when probes are loaded — a + /// quiet node is not a down sensor (the JEF-308 quiet≠blind invariant). + pub signals_emitted: u64, + /// When the window closed, as Unix epoch millis. Defaulted → the engine stamps ingest time. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_at_ms: Option, +} + +impl AgentReport { + /// Whether this report means the node is **blind** despite the agent being up: no probe + /// attached, so nothing is being observed. This is the Ready-but-blind failure mode + /// liveness-as-signal-flow catches (a pod-Ready check never would). + pub fn is_blind(&self) -> bool { + self.probes_loaded == 0 + } + + /// Whether the agent loaded only SOME of its probes — partial coverage (degraded, not blind). + /// False when fully loaded, or when blind (`probes_loaded == 0`, which reads as blind, not + /// partial), or when the build declares no probes at all (`probes_total == 0`). + pub fn is_partial(&self) -> bool { + self.probes_loaded > 0 && self.probes_total > 0 && self.probes_loaded < self.probes_total + } +} + #[cfg(test)] mod tests { use super::*; @@ -339,6 +392,7 @@ mod tests { attribution: Attribution::by_pod_uid("uid"), source: Some("protector-agent".into()), observed_at_ms: Some(1_710_000_000_000), + node: None, behavior: Behavior::SecretRead { secret: "app/session-key".into(), source: SecretReadSource::Mounted, @@ -553,6 +607,101 @@ mod tests { assert_eq!(serde_json::from_value::(v).unwrap(), w); } + #[test] + fn observation_carries_the_node_and_omits_it_when_absent() { + // JEF-308: the agent stamps its node so coverage is derivable PER NODE. When present it + // rides the wire; when absent (Falco, older agents) it is omitted — never guessed. + let with_node = RuntimeObservation { + attribution: Attribution::by_pod_uid("uid"), + source: Some("protector-agent".into()), + observed_at_ms: None, + node: Some("node-a".into()), + behavior: Behavior::ProcessExec { + path: "/bin/sh".into(), + }, + }; + let v = serde_json::to_value(&with_node).unwrap(); + assert_eq!(v["node"], serde_json::json!("node-a")); + assert_eq!( + serde_json::from_value::(v).unwrap(), + with_node + ); + + // Absent node ⇒ the key is omitted (byte-stable for node-agnostic sensors), and a legacy + // observation with no `node` deserializes back to `None`. + let no_node: RuntimeObservation = serde_json::from_value(serde_json::json!({ + "namespace": "app", "pod": "web", + "behavior": {"kind": "alert", "rule": "shell"} + })) + .unwrap(); + assert_eq!(no_node.node, None); + let reser = serde_json::to_value(&no_node).unwrap(); + assert!( + reser.get("node").is_none(), + "absent node is omitted on the wire" + ); + } + + #[test] + fn agent_report_round_trips_and_classifies_blind_vs_partial() { + // A healthy report: all probes loaded, some signals — round-trips. + let healthy = AgentReport { + node: "node-a".into(), + probes_loaded: 6, + probes_total: 6, + signals_emitted: 12, + observed_at_ms: Some(1_710_000_000_000), + }; + let v = serde_json::to_value(&healthy).unwrap(); + assert_eq!(serde_json::from_value::(v).unwrap(), healthy); + assert!(!healthy.is_blind()); + assert!(!healthy.is_partial()); + + // Quiet but healthy: probes loaded, zero signals — NOT blind, NOT partial (quiet≠blind). + let quiet = AgentReport { + signals_emitted: 0, + ..healthy.clone() + }; + assert!( + !quiet.is_blind(), + "a quiet node with probes loaded is not blind" + ); + assert!(!quiet.is_partial()); + + // Ready but blind: the agent is up but no probe attached — blind despite pod-Ready. + let blind = AgentReport { + probes_loaded: 0, + ..healthy.clone() + }; + assert!(blind.is_blind()); + assert!( + !blind.is_partial(), + "zero probes reads as blind, not partial" + ); + + // Partial: some but not all probes attached — degraded coverage. + let partial = AgentReport { + probes_loaded: 4, + ..healthy + }; + assert!(!partial.is_blind()); + assert!(partial.is_partial()); + } + + #[test] + fn agent_report_observed_at_ms_is_omitted_when_absent() { + let report = AgentReport { + node: "n".into(), + probes_loaded: 1, + probes_total: 1, + signals_emitted: 0, + observed_at_ms: None, + }; + let v = serde_json::to_value(&report).unwrap(); + assert!(v.get("observed_at_ms").is_none()); + assert_eq!(serde_json::from_value::(v).unwrap(), report); + } + #[test] fn only_alert_corroborates() { assert!(Behavior::Alert { rule: "x".into() }.is_alert()); diff --git a/charts/protector/templates/agent-daemonset.yaml b/charts/protector/templates/agent-daemonset.yaml index 259ef70..9705e41 100644 --- a/charts/protector/templates/agent-daemonset.yaml +++ b/charts/protector/templates/agent-daemonset.yaml @@ -68,6 +68,15 @@ spec: {{- toYaml .Values.agent.capabilities | nindent 16 }} {{- end }} env: + # The node this agent runs on (JEF-308), via the downward API. The agent stamps it + # onto every observation AND onto its per-node liveness beacon (/agent-liveness), so + # the engine can report runtime-corroboration coverage PER NODE ("blind on node X") + # rather than fleet-aggregate — and a Ready-but-blind agent (probes failed to load) + # still reads blind. Signal-flow liveness, not pod-Ready. + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName # The engine's behavioral ingest base — the agent appends /behavior. - name: PROTECTOR_AGENT_ENDPOINT value: "http://{{ include "protector.fullname" . }}-falco-ingest.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.engine.falco.port }}" diff --git a/engine/examples/dashboard_preview.rs b/engine/examples/dashboard_preview.rs index a032474..a5d9c4f 100644 --- a/engine/examples/dashboard_preview.rs +++ b/engine/examples/dashboard_preview.rs @@ -120,6 +120,7 @@ fn breach_finding() -> Finding { paths_truncated: false, evidence, recency: None, + node: None, } } @@ -140,6 +141,7 @@ fn simple_finding(entry: &str, objective: &str) -> Finding { paths_truncated: false, evidence: EntryEvidence::default(), recency: None, + node: None, } } @@ -172,6 +174,7 @@ fn redundant_finding() -> Finding { paths_truncated: false, evidence: EntryEvidence::default(), recency: None, + node: None, } } diff --git a/engine/src/engine/dashboard/admission_tests.rs b/engine/src/engine/dashboard/admission_tests.rs index 2dc90b3..91ad250 100644 --- a/engine/src/engine/dashboard/admission_tests.rs +++ b/engine/src/engine/dashboard/admission_tests.rs @@ -9,7 +9,9 @@ use std::time::SystemTime; use crate::engine::policy_log::PolicyDecisionRecord; -use crate::engine::state::{BakeStats, Finding, ModelHealth, ReadinessConfig, derive_readiness}; +use crate::engine::state::{ + BakeStats, Finding, ModelHealth, ReadinessConfig, RuntimeCoverage, derive_readiness, +}; use super::page; use super::view_model::{build_admission_view, build_status_strip}; @@ -27,7 +29,13 @@ fn judging_readiness() -> crate::engine::state::Readiness { }; let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - derive_readiness(&config, ModelHealth::Ok, &bake, Some(SystemTime::now())) + derive_readiness( + &config, + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ) } /// Build the persistent strip from a given findings snapshot (the strip the Admission view carries). diff --git a/engine/src/engine/dashboard/components/finding_detail.rs b/engine/src/engine/dashboard/components/finding_detail.rs index 3b1b38d..90eab38 100644 --- a/engine/src/engine/dashboard/components/finding_detail.rs +++ b/engine/src/engine/dashboard/components/finding_detail.rs @@ -23,6 +23,9 @@ pub(super) fn detail_panel(f: &FindingProps) -> Markup { } /// The verbatim model verdict — the model's own words first (brief: "why" is one click away). +/// A blind-node caveat (JEF-308), when present, is rendered LOUD (not muted): a propose-only +/// finding on a node with no live sensor must not read as reassuringly calm. The node name is +/// auto-escaped by maud (never `PreEscaped`). fn verdict_block(f: &FindingProps) -> Markup { html! { section.detail-section.verdict-block { @@ -31,6 +34,12 @@ fn verdict_block(f: &FindingProps) -> Markup { Some(v) => p.verdict-prose { (v) } None => p.verdict-prose.muted { "awaiting judgement \u{2014} the model has not judged this entry yet" } } + @if let Some(caveat) = &f.blind_node_caveat { + p.verdict-caveat.blind-node-caveat role="note" { + span.caveat-glyph aria-hidden="true" { "\u{26A0} " } // ⚠ + (caveat) + } + } } } } diff --git a/engine/src/engine/dashboard/components/readiness_view.rs b/engine/src/engine/dashboard/components/readiness_view.rs index b86fd74..183a115 100644 --- a/engine/src/engine/dashboard/components/readiness_view.rs +++ b/engine/src/engine/dashboard/components/readiness_view.rs @@ -7,7 +7,9 @@ use maud::{Markup, html}; -use crate::engine::dashboard::view_model::props::{ReadinessRowProps, ReadinessViewProps}; +use crate::engine::dashboard::view_model::props::{ + NodeRowProps, ReadinessRowProps, ReadinessViewProps, +}; /// Render the Readiness view: the coverage rows under the persistent strip (composed by `page`). pub fn readiness_view(v: &ReadinessViewProps) -> Markup { @@ -57,6 +59,12 @@ fn coverage_row(r: &ReadinessRowProps) -> Markup { } p.cov-detail.t-data { (r.detail) } p.cov-why.t-body.muted { (r.why) } + // The per-node runtime-corroboration breakdown (JEF-308) — a server-rendered table + // inside
(NO JS, per the maud/server-render convention). Only the + // runtime-corroboration row carries nodes; every other row skips this. + @if !r.nodes.is_empty() { + (node_breakdown(&r.nodes)) + } // The how-to-enable instruction: always shown when there is an env var/mount, and read // as an action ("enable with …") when the input is an absent weakening gap. @if !r.enable.is_empty() { @@ -71,3 +79,39 @@ fn coverage_row(r: &ReadinessRowProps) -> Markup { } } } + +/// The per-node runtime-corroboration breakdown (JEF-308): a server-rendered `` inside a +/// `
` disclosure — NO client JS (the maud/server-render convention). Each row is a node, +/// its honest state (colour + glyph + word, never colour alone), and a live detail. Node names are +/// UNTRUSTED — maud auto-escapes them (never `PreEscaped`). +fn node_breakdown(nodes: &[NodeRowProps]) -> Markup { + let blind = nodes.iter().filter(|n| n.state.token() == "blind").count(); + html! { + details.cov-nodes { + summary.cov-nodes-summary.t-micro { + (nodes.len()) " node" (if nodes.len() == 1 { "" } else { "s" }) + @if blind > 0 { " \u{2014} " (blind) " blind" } + } + table.cov-nodes-table { + thead { + tr { th { "node" } th { "state" } th { "detail" } } + } + tbody { + @for n in nodes { + tr data-state=(n.state.token()) { + td.cov-node-name { (n.node) } + td { + span class={ "node-state node-" (n.state.token()) } { + span.node-state-glyph aria-hidden="true" { (n.state.glyph()) } + " " + span.node-state-word { (n.state.word()) } + } + } + td.cov-node-detail { (n.detail) } + } + } + } + } + } + } +} diff --git a/engine/src/engine/dashboard/mod.rs b/engine/src/engine/dashboard/mod.rs index e093592..3135a57 100644 --- a/engine/src/engine/dashboard/mod.rs +++ b/engine/src/engine/dashboard/mod.rs @@ -81,7 +81,8 @@ impl DashboardState { let health: ModelHealth = self.findings.model_health(); let bake: BakeStats = self.findings.bake(); let last_pass: Option = self.findings.last_pass(); - derive_readiness(&config, health, &bake, last_pass) + let runtime = self.findings.runtime_coverage(); + derive_readiness(&config, health, &bake, last_pass, &runtime) } /// Build the persistent status strip carrying the TRUE findings counts (brief §3/§4). The @@ -271,3 +272,8 @@ mod path_view_tests; #[cfg(test)] mod admission_tests; + +// JEF-308: the runtime-corroboration per-node breakdown render + node-name escaping, in its own +// file to keep `tests.rs` under the 1,000-line cap (CLAUDE.md). +#[cfg(test)] +mod readiness_render_tests; diff --git a/engine/src/engine/dashboard/readiness_render_tests.rs b/engine/src/engine/dashboard/readiness_render_tests.rs new file mode 100644 index 0000000..bfd279c --- /dev/null +++ b/engine/src/engine/dashboard/readiness_render_tests.rs @@ -0,0 +1,75 @@ +//! Render-level tests for the JEF-308 runtime-corroboration row: the per-node breakdown is a +//! server-rendered `
` inside `
` (no JS), a blind node is surfaced loudly, and an +//! untrusted node name is ESCAPED (maud auto-escape — never `PreEscaped`). + +use std::time::SystemTime; + +use crate::engine::dashboard::page; +use crate::engine::dashboard::view_model::{build_readiness_view, build_status_strip}; +use crate::engine::state::{ + BakeStats, BlindReason, ModelHealth, NodeCoverage, NodeState, ReadinessConfig, RuntimeCoverage, + derive_readiness, +}; + +fn covered() -> ReadinessConfig { + ReadinessConfig { + model_attached: true, + kev_count: 5, + epss_count: 5, + journal_durable: true, + armed: false, + tuf_cache_age_secs: Some(60), + unverifiable_spike: false, + } +} + +#[test] +fn per_node_breakdown_is_a_server_table_and_escapes_node_names() { + // A malicious node name plus a healthy one — the row is degraded (one blind), named. + let coverage = RuntimeCoverage { + nodes: vec![ + NodeCoverage { + node: "node-a".into(), + state: NodeState::Healthy { signals: 2 }, + }, + NodeCoverage { + node: "".into(), + state: NodeState::Blind { + reason: BlindReason::NotReporting, + }, + }, + ], + }; + let readiness = derive_readiness( + &covered(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &coverage, + ); + let strip = build_status_strip("prod".into(), &[], &[], &readiness, Some(SystemTime::now())); + let v = build_readiness_view(strip, &readiness); + let html = page::readiness_page(&v).into_string(); + + assert!( + html.contains("Runtime corroboration"), + "the collapsed row label" + ); + assert!( + html.contains(" tag never reaches the HTML. + assert!( + !html.contains(""), + "an untrusted node name must be escaped, not injected" + ); + assert!( + html.contains("<script>"), + "it appears in escaped form" + ); + // A blind node is surfaced loudly, never quietly reassuring. + assert!(html.contains("BLIND")); +} diff --git a/engine/src/engine/dashboard/tests.rs b/engine/src/engine/dashboard/tests.rs index 99a751c..58145f9 100644 --- a/engine/src/engine/dashboard/tests.rs +++ b/engine/src/engine/dashboard/tests.rs @@ -7,7 +7,7 @@ use std::time::SystemTime; use crate::engine::reason::adjudicate::Verdict; use crate::engine::state::{ BakeStats, EntryEvidence, Finding, Judgement, LeftAloneEntry, ModelHealth, PathStep, Readiness, - ReadinessConfig, Report, ReversionRecord, WouldActEntry, derive_readiness, + ReadinessConfig, Report, ReversionRecord, RuntimeCoverage, WouldActEntry, derive_readiness, }; use super::page; @@ -28,7 +28,13 @@ pub(super) fn judging_readiness() -> Readiness { }; let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - derive_readiness(&config, ModelHealth::Ok, &bake, Some(SystemTime::now())) + derive_readiness( + &config, + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ) } /// A readiness snapshot for a warming (no pass yet) engine — not honestly calm. @@ -38,6 +44,7 @@ fn warming_readiness() -> Readiness { ModelHealth::Unknown, &BakeStats::default(), None, + &RuntimeCoverage::default(), ) } @@ -57,6 +64,7 @@ fn timed_out_readiness() -> Readiness { ModelHealth::Timeout, &BakeStats::default(), Some(SystemTime::now()), + &RuntimeCoverage::default(), ) } @@ -79,6 +87,7 @@ pub(super) fn breach_finding(entry: &str, verdict: Verdict) -> Finding { paths_truncated: false, evidence: EntryEvidence::default(), recency: None, + node: None, } } @@ -829,7 +838,13 @@ fn readiness_view_renders_a_row_per_input_with_enable_instruction() { }; let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let readiness = derive_readiness(&config, ModelHealth::Ok, &bake, Some(SystemTime::now())); + let readiness = derive_readiness( + &config, + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ); let v = build_readiness_view(strip_from(&[]), &readiness); let html = page::readiness_page(&v).into_string(); assert!(html.contains("decision inputs"), "the section heading"); diff --git a/engine/src/engine/dashboard/view_model/findings.rs b/engine/src/engine/dashboard/view_model/findings.rs index 43010f4..c3688b6 100644 --- a/engine/src/engine/dashboard/view_model/findings.rs +++ b/engine/src/engine/dashboard/view_model/findings.rs @@ -3,9 +3,12 @@ //! brief §5) — corroborated-live → model-promoted → escalations → awaiting → cleared. This is //! the data layer: it touches `state::`/`graph::` domain types; the components never do. +use std::collections::HashSet; + use crate::engine::graph::{Behavior, NodeKey}; use crate::engine::state::{ - CveEvidence, EntryEvidence, Finding, FindingEvidence, Judgement, PathStep, + CveEvidence, EntryEvidence, Finding, FindingEvidence, Judgement, NodeCoverageState, PathStep, + Readiness, }; use super::posture::{delta_of, live_tag_of, posture_of}; @@ -241,9 +244,33 @@ fn judgement_props(entry: &str, judgements: &[Judgement]) -> JudgementProps { } } +/// The set of blind node names from the readiness snapshot (JEF-308) — the runtime-corroboration +/// row's per-node breakdown, filtered to the `Blind` state. Used to add the blind-node caveat to a +/// finding whose node has no live sensor. +pub(super) fn blind_nodes_of(readiness: &Readiness) -> HashSet { + readiness + .inputs + .iter() + .find(|r| r.id == "runtime-corroboration") + .map(|r| { + r.nodes + .iter() + .filter(|n| n.state == NodeCoverageState::Blind) + .map(|n| n.node.clone()) + .collect() + }) + .unwrap_or_default() +} + /// Map one [`Finding`] into its presentation props. `judgements` is the newest-first judgement /// snapshot, used to attach the verbatim prompt/reply for the "show model prompt" disclosure. -pub(super) fn finding_props(f: &Finding, judgements: &[Judgement]) -> FindingProps { +/// `blind_nodes` is the set of nodes with no live runtime sensor (JEF-308) — a finding on a blind +/// node that isn't corroborated carries a caveat so its calm propose-only reading isn't dishonest. +pub(super) fn finding_props( + f: &Finding, + judgements: &[Judgement], + blind_nodes: &HashSet, +) -> FindingProps { let posture = posture_of(f.verdict.as_ref()); let live_tag = live_tag_of(f.verdict.as_ref()); FindingProps { @@ -266,7 +293,26 @@ pub(super) fn finding_props(f: &Finding, judgements: &[Judgement]) -> FindingPro cut: f.cut.clone(), evidence: evidence_props(&f.evidence), judgement: judgement_props(&f.entry, judgements), + blind_node_caveat: blind_node_caveat(f, blind_nodes), + } +} + +/// The blind-node caveat for a finding (JEF-308), or `None`. Applies when the finding is NOT +/// live-corroborated AND runs on a node with no live sensor: its calm propose-only reading would be +/// dishonest there, because we can't see whether the path is being exploited — absence of a signal +/// is not evidence of safety. A corroborated finding already has a live signal, and a finding whose +/// node is unknown or sensored gets no caveat. +fn blind_node_caveat(f: &Finding, blind_nodes: &HashSet) -> Option { + if f.corroborated { + return None; + } + let node = f.node.as_ref()?; + if !blind_nodes.contains(node) { + return None; } + Some(format!( + "no live runtime sensor on node {node} \u{2014} absence of a signal here is not evidence of safety" + )) } /// The urgency rank for the sort (lower = MORE urgent). Urgency is NOT severity (ADR-0016): a @@ -488,11 +534,15 @@ fn collapse_pod_replicas(mut rows: Vec) -> Vec { /// findings are surfaced — the caller passes the breach-relevant set. Pod-replica collapse and /// fan-out collapse run first, then the urgency sort (stable within a rank, by entry for /// determinism). -pub(super) fn map_findings(findings: &[Finding], judgements: &[Judgement]) -> Vec { +pub(super) fn map_findings( + findings: &[Finding], + judgements: &[Judgement], + blind_nodes: &HashSet, +) -> Vec { let mut rows: Vec = findings .iter() .filter(|f| f.breach_relevant) - .map(|f| finding_props(f, judgements)) + .map(|f| finding_props(f, judgements, blind_nodes)) .collect(); rows = collapse_pod_replicas(rows); rows = collapse_fanout(rows); diff --git a/engine/src/engine/dashboard/view_model/findings/tests.rs b/engine/src/engine/dashboard/view_model/findings/tests.rs index a58735a..a79fb11 100644 --- a/engine/src/engine/dashboard/view_model/findings/tests.rs +++ b/engine/src/engine/dashboard/view_model/findings/tests.rs @@ -4,10 +4,17 @@ use super::*; use crate::engine::dashboard::view_model::props::DeltaProps; use crate::engine::reason::adjudicate::Verdict; +use std::collections::HashSet; + use crate::engine::state::{ CveEvidence, Delta, EntryEvidence, Finding, Judgement, PathStep, RecencyInfo, }; +/// An empty blind-node set — most mapping tests aren't exercising the JEF-308 caveat path. +fn no_blind() -> HashSet { + HashSet::new() +} + /// A minimal breach-relevant finding with a typed verdict, for the mapping tests. fn finding(entry: &str, objective: &str, verdict: Option) -> Finding { Finding { @@ -28,9 +35,45 @@ fn finding(entry: &str, objective: &str, verdict: Option) -> Finding { paths_truncated: false, evidence: EntryEvidence::default(), recency: None, + node: None, } } +/// JEF-308: a latent / propose-only finding on a BLIND node carries the "no live sensor" caveat — +/// its calm propose-only reading would be dishonest, so the detail says absence of a signal isn't +/// evidence of safety. A corroborated finding, or one on a sensored node, gets no caveat. +#[test] +fn latent_finding_on_a_blind_node_carries_the_caveat() { + let mut latent = finding("workload/app/Pod/web-1", "secret/app/db", None); + latent.disposition = "latent foothold — propose".into(); + latent.corroborated = false; + latent.node = Some("node-b".into()); + let blind: HashSet = ["node-b".to_string()].into_iter().collect(); + + let props = finding_props(&latent, &[], &blind); + let caveat = props + .blind_node_caveat + .expect("a latent finding on a blind node carries the caveat"); + assert!(caveat.contains("node-b")); + assert!(caveat.contains("not evidence of safety")); + + // The SAME finding on a sensored node (not in the blind set) gets no caveat. + assert!( + finding_props(&latent, &[], &no_blind()) + .blind_node_caveat + .is_none() + ); + + // A live-corroborated finding on the same blind node gets no caveat (it has a live signal). + let mut corroborated = latent.clone(); + corroborated.corroborated = true; + assert!( + finding_props(&corroborated, &[], &blind) + .blind_node_caveat + .is_none() + ); +} + #[test] fn only_breach_relevant_rows_are_surfaced() { let mut keep = finding( @@ -41,7 +84,7 @@ fn only_breach_relevant_rows_are_surfaced() { keep.foothold = true; let mut drop = finding("workload/app/Pod/x", "secret/app/y", None); drop.breach_relevant = false; - let rows = map_findings(&[keep, drop], &[]); + let rows = map_findings(&[keep, drop], &[], &no_blind()); assert_eq!(rows.len(), 1); } @@ -59,7 +102,7 @@ fn urgency_sort_puts_live_breach_first_cleared_last() { Some(Verdict::Refuted("internal".into())), ); let awaiting = finding("endpoint/d", "secret/x", None); - let rows = map_findings(&[cleared, awaiting, promoted, live], &[]); + let rows = map_findings(&[cleared, awaiting, promoted, live], &[], &no_blind()); let order: Vec = rows.iter().map(|r| r.posture).collect(); assert_eq!( order, @@ -87,7 +130,7 @@ fn escalation_outranks_awaiting() { age_secs: Some(5), }); let awaiting = finding("endpoint/a", "secret/x", None); - let rows = map_findings(&[awaiting, escalated], &[]); + let rows = map_findings(&[awaiting, escalated], &[], &no_blind()); // The escalation (rank 2) sorts before the awaiting row (rank 4). assert!(matches!(rows[0].delta, DeltaProps::Escalated)); } @@ -117,7 +160,7 @@ fn evidence_summary_counts_and_kev() { title: None, }, ]; - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); assert_eq!(rows[0].evidence_summary.cve_count, 2); assert!(rows[0].evidence_summary.kev); assert!(!rows[0].evidence.is_empty()); @@ -126,7 +169,7 @@ fn evidence_summary_counts_and_kev() { #[test] fn empty_evidence_is_flagged_empty() { let f = finding("endpoint/a", "secret/x", None); - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); assert!(rows[0].evidence.is_empty()); assert!(rows[0].evidence_summary.is_empty()); } @@ -142,7 +185,7 @@ fn fanout_collapses_a_high_objective_entry() { Some(Verdict::Refuted("cleared".into())), )); } - let mapped = map_findings(&rows, &[]); + let mapped = map_findings(&rows, &[], &no_blind()); let argo: Vec<_> = mapped.iter().filter(|r| r.entry == "argocd").collect(); assert_eq!(argo.len(), 1, "fan-out collapses to one row"); assert_eq!(argo[0].fanout, Some(20)); @@ -158,7 +201,7 @@ fn judgement_prompt_is_attached_by_entry() { prompt: Some("the prompt".into()), reply: Some("the reply".into()), }]; - let rows = map_findings(&[f], &judgements); + let rows = map_findings(&[f], &judgements, &no_blind()); assert_eq!(rows[0].judgement.prompt.as_deref(), Some("the prompt")); assert_eq!(rows[0].judgement.reply.as_deref(), Some("the reply")); } @@ -185,7 +228,7 @@ fn path_props_carry_node_glyphs_and_mark_the_cut() { }, ]; f.cut = Some("deployment/edge/api -[reaches/Tcp/5432]-> statefulset/app/db".into()); - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); let path = &rows[0].path; assert_eq!(path.len(), 2); // The foothold entry node reads as the internet front door (🌐), not its bare kind. @@ -230,7 +273,7 @@ fn multi_path_marks_edges_shared_across_all_paths() { }, ]; f.paths = vec![path_a, path_b]; - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); let paths = &rows[0].paths; assert_eq!(paths.len(), 2, "both proven paths are mapped, not just one"); // The common first hop is a shared bottleneck in both routes... @@ -252,7 +295,7 @@ fn single_path_finding_falls_back_and_marks_no_shared_edge() { // A lone-path finding (the common case) sets no `paths`; the mapper falls back to the // representative path so a chain always renders, and nothing is marked shared. let f = finding("endpoint/a", "secret/x", Some(Verdict::Confirmed)); - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); assert_eq!( rows[0].paths.len(), 1, @@ -268,7 +311,7 @@ fn verdict_summary_is_the_models_verbatim_words() { "secret/x", Some(Verdict::Exploitable("CVE reachable".into())), ); - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); assert_eq!( rows[0].verdict_summary.as_deref(), Some("exploitable — CVE reachable") @@ -287,7 +330,7 @@ fn distinct_entries_that_slugify_alike_get_distinct_ids() { // non-alphanumeric → `-` rule, so the old finding_id collided. The hash suffix separates them. let a = finding("secret/app/db", "secret/x", Some(Verdict::Confirmed)); let b = finding("secret-app-db", "secret/x", Some(Verdict::Confirmed)); - let rows = map_findings(&[a, b], &[]); + let rows = map_findings(&[a, b], &[], &no_blind()); assert_eq!(rows.len(), 2, "two distinct entries stay two rows"); assert_ne!( rows[0].id, rows[1].id, @@ -300,8 +343,8 @@ fn finding_id_is_stable_across_renders() { // The id must be deterministic so the JS's persisted open-state keying survives a poll swap. let f1 = finding("endpoint/a", "secret/x", Some(Verdict::Confirmed)); let f2 = finding("endpoint/a", "secret/x", Some(Verdict::Confirmed)); - let r1 = map_findings(&[f1], &[]); - let r2 = map_findings(&[f2], &[]); + let r1 = map_findings(&[f1], &[], &no_blind()); + let r2 = map_findings(&[f2], &[], &no_blind()); assert_eq!( r1[0].id, r2[0].id, "the same entry always yields the same id" @@ -332,7 +375,7 @@ fn statefulset_replicas_collapse_to_one_row_with_worst_posture() { "secret/analytics/db", Some(Verdict::Refuted("internal".into())), ); - let rows = map_findings(&[r0, r1, r2], &[]); + let rows = map_findings(&[r0, r1, r2], &[], &no_blind()); assert_eq!(rows.len(), 1, "three replicas collapse to one row"); assert_eq!(rows[0].replicas, Some(3), "the row carries the ×3 count"); assert_eq!( @@ -359,7 +402,7 @@ fn deployment_replicas_collapse_by_owning_workload() { "secret/web/session", Some(Verdict::Refuted("internal".into())), ); - let rows = map_findings(&[a, b], &[]); + let rows = map_findings(&[a, b], &[], &no_blind()); assert_eq!(rows.len(), 1, "two deployment replicas collapse to one row"); assert_eq!(rows[0].replicas, Some(2)); assert_eq!(rows[0].entry, "web/storefront"); @@ -378,7 +421,7 @@ fn unrelated_pods_do_not_merge() { "secret/web/cart", Some(Verdict::Confirmed), ); - let rows = map_findings(&[a, b], &[]); + let rows = map_findings(&[a, b], &[], &no_blind()); assert_eq!(rows.len(), 2, "unrelated pods are never merged"); assert!(rows.iter().all(|r| r.replicas.is_none())); } @@ -391,7 +434,7 @@ fn standalone_pod_stays_a_single_row() { "secret/ops/kubeconfig", Some(Verdict::Confirmed), ); - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); assert_eq!(rows.len(), 1); assert_eq!( rows[0].replicas, None, @@ -408,7 +451,7 @@ fn a_single_replica_named_pod_is_not_collapsed() { "secret/web/x", Some(Verdict::Confirmed), ); - let rows = map_findings(&[f], &[]); + let rows = map_findings(&[f], &[], &no_blind()); assert_eq!(rows.len(), 1); assert_eq!(rows[0].replicas, None, "a lone pod is not a replica set"); } diff --git a/engine/src/engine/dashboard/view_model/mod.rs b/engine/src/engine/dashboard/view_model/mod.rs index 4d50dda..71e3020 100644 --- a/engine/src/engine/dashboard/view_model/mod.rs +++ b/engine/src/engine/dashboard/view_model/mod.rs @@ -40,7 +40,10 @@ fn strip_from_findings( readiness: &Readiness, last_pass: Option, ) -> (StatusStripProps, Vec) { - let rows = findings::map_findings(findings, judgements); + // Blind nodes (JEF-308) come from the readiness runtime-corroboration breakdown, so a finding + // on a node with no live sensor carries its caveat. + let blind_nodes = findings::blind_nodes_of(readiness); + let rows = findings::map_findings(findings, judgements, &blind_nodes); let breach = rows.iter().filter(|r| r.posture == Posture::Breach).count(); let awaiting = rows .iter() diff --git a/engine/src/engine/dashboard/view_model/props/mod.rs b/engine/src/engine/dashboard/view_model/props/mod.rs index 4ea437d..4ecd924 100644 --- a/engine/src/engine/dashboard/view_model/props/mod.rs +++ b/engine/src/engine/dashboard/view_model/props/mod.rs @@ -287,6 +287,12 @@ pub struct FindingProps { pub cut: Option, pub evidence: EvidenceProps, pub judgement: JudgementProps, + /// The blind-node caveat (JEF-308): set when this finding sits on a node with NO live runtime + /// sensor and its disposition is latent / propose-only (uncorroborated). Its calm propose-only + /// reading would be dishonest there — absence of a corroborating signal is not evidence of + /// safety — so the detail renders this caveat. `None` when the node has a live sensor, the + /// finding is corroborated, or the node isn't known. + pub blind_node_caveat: Option, } /// The three honesty axes the status strip carries (brief §3): decided/judging/covered. Never @@ -514,6 +520,67 @@ pub struct ReadinessRowProps { pub detail: String, /// Whether this input being absent WEAKENS the model's decision. pub weakens_decisions: bool, + /// The per-node runtime-corroboration breakdown (JEF-308) — populated ONLY for the + /// `runtime-corroboration` row, empty otherwise. Rendered as a server-side `
` inside + /// `
` (no JS) so an operator can see exactly which node is blind. + pub nodes: Vec, +} + +/// One node's line in the runtime-corroboration per-node breakdown (JEF-308). Every string is +/// UNTRUSTED at render (the node name can be attacker-influenced) — the component auto-escapes it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeRowProps { + /// The node name (untrusted). + pub node: String, + pub state: NodeCoverageStateProps, + /// A short live detail (signal count, "quiet", probe fraction, or the blind reason). + pub detail: String, +} + +/// One expected node's honest liveness state (JEF-308) — colour + glyph + word, never colour alone. +/// "Quiet" and "blind" never collapse into one word: a quiet-but-healthy node is not a down sensor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeCoverageStateProps { + /// Reporting with probes loaded (quiet counts). + Healthy, + /// Reporting but only some probes attached. + Degraded, + /// No live sensor on this expected node — the attention case. + Blind, + /// A node the agent isn't scheduled on — out-of-scope, not blind. + OutOfScope, +} + +impl NodeCoverageStateProps { + /// The CSS token suffix (`--node-{kind}`) / `data-state` value. + pub fn token(self) -> &'static str { + match self { + NodeCoverageStateProps::Healthy => "healthy", + NodeCoverageStateProps::Degraded => "degraded", + NodeCoverageStateProps::Blind => "blind", + NodeCoverageStateProps::OutOfScope => "out-of-scope", + } + } + + /// The glyph carrying the state without colour. + pub fn glyph(self) -> &'static str { + match self { + NodeCoverageStateProps::Healthy => "\u{2713}", // ✓ + NodeCoverageStateProps::Degraded => "\u{25D0}", // ◐ + NodeCoverageStateProps::Blind => "\u{25CF}", // ● + NodeCoverageStateProps::OutOfScope => "\u{2014}", // — + } + } + + /// The word — always present alongside colour + glyph. + pub fn word(self) -> &'static str { + match self { + NodeCoverageStateProps::Healthy => "healthy", + NodeCoverageStateProps::Degraded => "degraded", + NodeCoverageStateProps::Blind => "BLIND", + NodeCoverageStateProps::OutOfScope => "out-of-scope", + } + } } /// The whole Readiness view's props: the persistent strip + the coverage rows, weakening-inputs diff --git a/engine/src/engine/dashboard/view_model/readiness.rs b/engine/src/engine/dashboard/view_model/readiness.rs index 272c9b9..fb12a92 100644 --- a/engine/src/engine/dashboard/view_model/readiness.rs +++ b/engine/src/engine/dashboard/view_model/readiness.rs @@ -4,9 +4,11 @@ //! the env var to enable it. Rows that WEAKEN decisions when absent float to the top so the gaps //! that demote the model's call are seen first. Data layer: touches `state::`; components never do. -use crate::engine::state::{InputState, Readiness, ReadinessRow}; +use crate::engine::state::{InputState, NodeCoverageState, Readiness, ReadinessRow}; -use super::props::{InputStateProps, ReadinessRowProps, ReadinessViewProps}; +use super::props::{ + InputStateProps, NodeCoverageStateProps, NodeRowProps, ReadinessRowProps, ReadinessViewProps, +}; /// Map the engine's [`InputState`] into the presentation enum (the honesty stays: Absent and /// Degraded never read as covered). @@ -18,7 +20,17 @@ fn input_state(state: InputState) -> InputStateProps { } } -/// Project one engine readiness row into its props. +/// Map an engine node-coverage state into its presentation enum (JEF-308). +fn node_state(state: NodeCoverageState) -> NodeCoverageStateProps { + match state { + NodeCoverageState::Healthy => NodeCoverageStateProps::Healthy, + NodeCoverageState::Degraded => NodeCoverageStateProps::Degraded, + NodeCoverageState::Blind => NodeCoverageStateProps::Blind, + NodeCoverageState::OutOfScope => NodeCoverageStateProps::OutOfScope, + } +} + +/// Project one engine readiness row into its props, carrying any per-node breakdown (JEF-308). fn row_props(row: &ReadinessRow) -> ReadinessRowProps { ReadinessRowProps { id: row.id.to_string(), @@ -28,6 +40,15 @@ fn row_props(row: &ReadinessRow) -> ReadinessRowProps { enable: row.enable.to_string(), detail: row.detail.clone(), weakens_decisions: row.weakens_decisions, + nodes: row + .nodes + .iter() + .map(|n| NodeRowProps { + node: n.node.clone(), + state: node_state(n.state), + detail: n.detail.clone(), + }) + .collect(), } } diff --git a/engine/src/engine/dashboard/view_model/readiness/tests.rs b/engine/src/engine/dashboard/view_model/readiness/tests.rs index bcc228f..b163788 100644 --- a/engine/src/engine/dashboard/view_model/readiness/tests.rs +++ b/engine/src/engine/dashboard/view_model/readiness/tests.rs @@ -3,7 +3,9 @@ use super::*; use crate::engine::dashboard::view_model::props::InputStateProps; -use crate::engine::state::{BakeStats, ModelHealth, ReadinessConfig, derive_readiness}; +use crate::engine::state::{ + BakeStats, ModelHealth, ReadinessConfig, RuntimeCoverage, derive_readiness, +}; use std::time::SystemTime; fn covered() -> ReadinessConfig { @@ -22,10 +24,17 @@ fn covered() -> ReadinessConfig { fn every_input_becomes_a_row_with_state_word_and_why() { let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let r = derive_readiness(&covered(), ModelHealth::Ok, &bake, Some(SystemTime::now())); + let r = derive_readiness( + &covered(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ); let rows = map_readiness(&r); - // model / kev / epss / falco / ebpf-agent / journal / tuf-root / arm-state == 8 inputs. - assert_eq!(rows.len(), 8); + // model / kev / epss / runtime-corroboration / journal / tuf-root / arm-state == 7 inputs + // (the former falco + ebpf-agent rows are collapsed into one runtime-corroboration row, JEF-308). + assert_eq!(rows.len(), 7); // Every row carries a non-empty label + why + state word (meaning never by colour alone). for row in &rows { assert!(!row.label.is_empty()); @@ -42,7 +51,13 @@ fn an_absent_weakening_input_floats_to_the_top() { config.kev_count = 0; let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let r = derive_readiness(&config, ModelHealth::Ok, &bake, Some(SystemTime::now())); + let r = derive_readiness( + &config, + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ); let rows = map_readiness(&r); let kev_pos = rows.iter().position(|row| row.id == "kev").unwrap(); let model_pos = rows.iter().position(|row| row.id == "model").unwrap(); @@ -65,6 +80,7 @@ fn a_degraded_model_reads_degraded_not_absent_or_present() { ModelHealth::Timeout, &BakeStats::default(), Some(SystemTime::now()), + &RuntimeCoverage::default(), ); let rows = map_readiness(&r); let model = rows.iter().find(|row| row.id == "model").unwrap(); @@ -79,6 +95,7 @@ fn arm_state_is_present_and_never_weakens() { ModelHealth::Ok, &BakeStats::default(), Some(SystemTime::now()), + &RuntimeCoverage::default(), ); let rows = map_readiness(&r); let arm = rows.iter().find(|row| row.id == "arm-state").unwrap(); diff --git a/engine/src/engine/dashboard/view_model/strip.rs b/engine/src/engine/dashboard/view_model/strip.rs index e9f96a4..30a90e5 100644 --- a/engine/src/engine/dashboard/view_model/strip.rs +++ b/engine/src/engine/dashboard/view_model/strip.rs @@ -13,11 +13,13 @@ use super::props::{CoverageChip, StatusStripProps}; /// The enrichment feeds shown as coverage chips in the strip, in a stable order. Arm-state and /// journal are reported elsewhere (the mode pill / Readiness tab), not as coverage chips. -const COVERAGE_FEEDS: [(&str, &str); 4] = [ +const COVERAGE_FEEDS: [(&str, &str); 3] = [ ("kev", "KEV"), ("epss", "EPSS"), - ("falco", "Falco"), - ("ebpf-agent", "eBPF"), + // ONE agent-sourced runtime-corroboration chip (JEF-308), replacing the former Falco + eBPF + // pair. It goes degraded the moment any expected node is blind (the collapsed readiness row's + // Degraded state), and reads as a gap (not present) when corroboration is wholly blind. + ("runtime-corroboration", "Runtime"), ]; /// Build the coverage chips from the readiness rows. A `Present` row is covered; a `Degraded` @@ -94,7 +96,10 @@ fn last_pass_age(at: SystemTime) -> String { #[cfg(test)] mod tests { use super::*; - use crate::engine::state::{BakeStats, ModelHealth, ReadinessConfig, derive_readiness}; + use crate::engine::state::{ + BakeStats, BlindReason, ModelHealth, NodeCoverage, NodeState, ReadinessConfig, + RuntimeCoverage, derive_readiness, + }; fn covered() -> ReadinessConfig { ReadinessConfig { @@ -108,22 +113,83 @@ mod tests { } } + /// A coverage snapshot from `(node, state)` pairs. + fn coverage(nodes: &[(&str, NodeState)]) -> RuntimeCoverage { + RuntimeCoverage { + nodes: nodes + .iter() + .map(|(n, s)| NodeCoverage { + node: (*n).to_string(), + state: *s, + }) + .collect(), + } + } + #[test] fn judging_model_strip_is_honest_calm() { let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let r = derive_readiness(&covered(), ModelHealth::Ok, &bake, Some(SystemTime::now())); + // One healthy agent node → the Runtime chip is present. + let cov = coverage(&[("node-a", NodeState::Healthy { signals: 2 })]); + let r = derive_readiness( + &covered(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &cov, + ); // Judging + covered + nothing breach/awaiting/uncertain (3 cleared) ⇒ honest all-clear. let strip = status_strip("prod".into(), &r, Some(SystemTime::now()), 0, 0, 0, 3, 0); assert!(strip.model_is_up()); assert!(strip.all_clear()); assert!(!strip.watching()); assert!(!strip.armed); - // KEV/EPSS present; Falco present (alert signal); eBPF absent. - let falco = strip.coverage.iter().find(|c| c.label == "Falco").unwrap(); - assert!(falco.present); - let ebpf = strip.coverage.iter().find(|c| c.label == "eBPF").unwrap(); - assert!(!ebpf.present); + // The former Falco + eBPF chips are collapsed into ONE agent-sourced Runtime chip. + assert!(strip.coverage.iter().all(|c| c.label != "Falco")); + assert!(strip.coverage.iter().all(|c| c.label != "eBPF")); + let runtime = strip + .coverage + .iter() + .find(|c| c.label == "Runtime") + .unwrap(); + assert!(runtime.present); + assert!(!runtime.degraded); + } + + /// JEF-308: the Runtime chip goes degraded the moment any expected node is blind. + #[test] + fn a_blind_node_degrades_the_runtime_chip() { + let cov = coverage(&[ + ("node-a", NodeState::Healthy { signals: 1 }), + ( + "node-b", + NodeState::Blind { + reason: BlindReason::NotReporting, + }, + ), + ]); + let r = derive_readiness( + &covered(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &cov, + ); + let strip = status_strip("prod".into(), &r, Some(SystemTime::now()), 0, 0, 0, 3, 0); + let runtime = strip + .coverage + .iter() + .find(|c| c.label == "Runtime") + .unwrap(); + assert!( + runtime.degraded, + "a blind node degrades the corroboration chip" + ); + assert!( + !strip.all_clear(), + "a degraded feed forbids the green all-clear" + ); } #[test] @@ -133,6 +199,7 @@ mod tests { ModelHealth::Timeout, &BakeStats::default(), Some(SystemTime::now()), + &RuntimeCoverage::default(), ); let strip = status_strip("prod".into(), &r, Some(SystemTime::now()), 0, 0, 0, 0, 0); assert!(!strip.model_is_up()); @@ -148,6 +215,7 @@ mod tests { ModelHealth::Unknown, &BakeStats::default(), None, + &RuntimeCoverage::default(), ); let strip = status_strip("prod".into(), &r, None, 0, 0, 0, 0, 0); assert!(!strip.model_is_up()); @@ -162,7 +230,13 @@ mod tests { fn established_signing_regression_forbids_green_and_watching() { let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let r = derive_readiness(&covered(), ModelHealth::Ok, &bake, Some(SystemTime::now())); + let r = derive_readiness( + &covered(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ); // Everything else is clear (would be all-clear) — but one established regression stands. let strip = status_strip("prod".into(), &r, Some(SystemTime::now()), 0, 0, 0, 3, 0) .with_signing_regressions(1, 0); @@ -183,7 +257,13 @@ mod tests { fn cold_signing_regression_is_watching_not_green() { let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let r = derive_readiness(&covered(), ModelHealth::Ok, &bake, Some(SystemTime::now())); + let r = derive_readiness( + &covered(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ); let strip = status_strip("prod".into(), &r, Some(SystemTime::now()), 0, 0, 0, 3, 0) .with_signing_regressions(0, 1); assert!(!strip.all_clear(), "a cold regression still forbids green"); @@ -199,7 +279,13 @@ mod tests { fn judging_with_pending_entries_is_watching_not_all_clear() { let mut bake = BakeStats::default(); bake.signals_by_variant.insert("alert".into(), 1); - let r = derive_readiness(&covered(), ModelHealth::Ok, &bake, Some(SystemTime::now())); + let r = derive_readiness( + &covered(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &RuntimeCoverage::default(), + ); // One entry still awaiting, one still uncertain — model hasn't cleared everything. let strip = status_strip("prod".into(), &r, Some(SystemTime::now()), 0, 1, 1, 4, 0); assert!(strip.model_is_up()); diff --git a/engine/src/engine/metrics.rs b/engine/src/engine/metrics.rs new file mode 100644 index 0000000..4e36370 --- /dev/null +++ b/engine/src/engine/metrics.rs @@ -0,0 +1,159 @@ +//! The engine's OTLP instruments (extracted from `mod.rs` to keep it under the 1,000-line cap, +//! CLAUDE.md). A cohesive unit: the counters/gauges/histograms the engine records against the +//! global meter, and their one-shot construction. `pub(super)` so the engine core reads the fields. + +/// OTLP instruments for the engine, recorded against the global meter (see +/// [`crate::telemetry`]). When no OTLP endpoint is configured the global meter is a +/// no-op, so these calls cost nothing — the engine is instrumented unconditionally. +/// Counters are cumulative; gauges hold the last pass's snapshot. +pub(super) struct EngineMetrics { + /// Process passes (one per observed change). + pub(super) passes: opentelemetry::metrics::Counter, + /// Adjudicator model invocations, by `result` (`ok`/`unavailable`). + pub(super) model_calls: opentelemetry::metrics::Counter, + /// Mitigations actuated, by `action` (`applied`/`reverted`). + pub(super) mitigations: opentelemetry::metrics::Counter, + /// Proven chains in the last pass. + pub(super) chains: opentelemetry::metrics::Gauge, + /// Breach-relevant findings (internet-facing) in the last pass. + pub(super) breach_paths: opentelemetry::metrics::Gauge, + /// Active mitigations currently in the ledger. + pub(super) active_mitigations: opentelemetry::metrics::Gauge, + /// Breach-path count by model `verdict` (the current judgement distribution). + pub(super) verdicts: opentelemetry::metrics::Gauge, + /// Behavioral signals ingested this pass, by `behavior` variant (alert/connection/ + /// secret-read/library-load/file-read/priv-change/exec) — the shadow-bake (JEF-48) + /// view of *what* the behavioral port is seeing, labeled low-cardinality (variant + /// names only, never per-pod). + pub(super) signals: opentelemetry::metrics::Counter, + /// Signal attribution outcome, by `outcome` (`resolved`/`unresolved`): how many + /// ingested signals the runtime adapter could attribute to a live workload vs drop as + /// an unknown cgroup UID. A sustained `unresolved` means the agent's UIDs aren't + /// matching pod metadata. + pub(super) attribution: opentelemetry::metrics::Counter, + /// `RuntimeEvents` store cardinality (distinct live observations) as of this pass — + /// a gauge so the TTL'd store's working-set size is observable. + pub(super) runtime_store: opentelemetry::metrics::Gauge, + /// Corroborations fired this pass: proven breach-relevant chains whose `corroborated` + /// predicate is set (ADR-0009). In shadow this is the countable answer to "would this + /// have promoted?" without any behavior change. + pub(super) corroborations: opentelemetry::metrics::Counter, + /// Per-pass adjudications that issued a fresh model call (verdict-cache miss). A + /// proper cumulative counter (replaces the prior `verdicts{verdict="judged_this_pass"}` + /// gauge hack) so model-call frequency is rate-able. + pub(super) judged: opentelemetry::metrics::Counter, + /// Per-pass adjudications served from the verdict cache (cache hit). Cumulative + /// counter, the companion to [`Self::judged`]. + pub(super) cached: opentelemetry::metrics::Counter, + /// Per-pass cache MISSES the engine declined to send to the model because the entry + /// (or the whole fleet, via the global breaker) was in inconclusive-adjudication + /// backoff (JEF-234). A cumulative counter: a sustained nonzero rate means the model is + /// degraded and the engine is correctly NOT hammering it (the bounding is working). + pub(super) skipped: opentelemetry::metrics::Counter, + /// Adjudicator model-call latency in milliseconds (histogram), recorded around each + /// fresh `judge` call so the slow CPU model's tail is visible. + pub(super) model_latency_ms: opentelemetry::metrics::Histogram, + /// Shadow would-have-acted report headline (JEF-143): distinct workloads the engine + /// WOULD have isolated over the default rolling window, as of this pass — the gates- + /// exiting-shadow figure (JEF-50), mirrored to OTLP like the bake counts. A gauge: + /// the current window snapshot, the in-process mirror of the would-have-acted report + /// aggregation. + pub(super) report_would_act: opentelemetry::metrics::Gauge, + /// Shadow report headline: distinct proven-but-cleared paths the model left alone + /// over the window (the trust half of the diff). + pub(super) report_left_alone: opentelemetry::metrics::Gauge, + /// Shadow report headline: would-acts whose projected cut was short-lived (likely + /// false positives) — the subset to discount when judging the shadow bake. + pub(super) report_short_lived: opentelemetry::metrics::Gauge, + /// Shadow report headline: would-acts made during an enrichment-coverage gap (no CVE + /// backing) — the ones to scrutinize first. + pub(super) report_coverage_gap: opentelemetry::metrics::Gauge, +} + +impl EngineMetrics { + pub(super) fn new() -> Self { + let m = opentelemetry::global::meter("protector.engine"); + Self { + passes: m + .u64_counter("protector.engine.passes") + .with_description("Engine process passes (one per observed change).") + .build(), + model_calls: m + .u64_counter("protector.engine.model_calls") + .with_description("Adjudicator model invocations by result.") + .build(), + mitigations: m + .u64_counter("protector.engine.mitigations") + .with_description("Mitigations actuated by action.") + .build(), + chains: m + .u64_gauge("protector.engine.chains") + .with_description("Proven chains in the last pass.") + .build(), + breach_paths: m + .u64_gauge("protector.engine.breach_paths") + .with_description("Breach-relevant (internet-facing) findings in the last pass.") + .build(), + active_mitigations: m + .u64_gauge("protector.engine.active_mitigations") + .with_description("Active mitigations in the ledger.") + .build(), + verdicts: m + .u64_gauge("protector.engine.verdicts") + .with_description("Breach paths by model verdict (current distribution).") + .build(), + signals: m + .u64_counter("protector.engine.signals") + .with_description("Behavioral signals ingested by variant.") + .build(), + attribution: m + .u64_counter("protector.engine.attribution") + .with_description("Signal attribution outcome (resolved/unresolved).") + .build(), + runtime_store: m + .u64_gauge("protector.engine.runtime_store") + .with_description("RuntimeEvents store cardinality (live observations).") + .build(), + corroborations: m + .u64_counter("protector.engine.corroborations") + .with_description("Corroborations fired (corroborated breach chains) per pass.") + .build(), + judged: m + .u64_counter("protector.engine.judged") + .with_description("Adjudications that issued a fresh model call (cache miss).") + .build(), + cached: m + .u64_counter("protector.engine.cached") + .with_description("Adjudications served from the verdict cache (cache hit).") + .build(), + skipped: m + .u64_counter("protector.engine.skipped") + .with_description( + "Adjudications skipped (inconclusive-adjudication backoff / breaker open).", + ) + .build(), + model_latency_ms: m + .f64_histogram("protector.engine.model_latency_ms") + .with_description("Adjudicator model-call latency in milliseconds.") + .build(), + report_would_act: m + .u64_gauge("protector.engine.report_would_act") + .with_description( + "Shadow report: workloads that would have been isolated (window).", + ) + .build(), + report_left_alone: m + .u64_gauge("protector.engine.report_left_alone") + .with_description("Shadow report: proven-but-cleared paths left alone (window).") + .build(), + report_short_lived: m + .u64_gauge("protector.engine.report_short_lived") + .with_description("Shadow report: short-lived would-acts (likely false positives).") + .build(), + report_coverage_gap: m + .u64_gauge("protector.engine.report_coverage_gap") + .with_description("Shadow report: would-acts during an enrichment-coverage gap.") + .build(), + } + } +} diff --git a/engine/src/engine/mod.rs b/engine/src/engine/mod.rs index 0c2f0d2..f9353d2 100644 --- a/engine/src/engine/mod.rs +++ b/engine/src/engine/mod.rs @@ -48,6 +48,10 @@ pub mod respond; // would-have-acted / readiness aggregations the per-pass OTLP mirror reads. pub mod state; +// OTLP instruments (extracted for the file-size cap, CLAUDE.md). +mod metrics; +use metrics::EngineMetrics; + use graph::delta::GraphSnapshot; use observe::Snapshot; use observe::adapter::Adapter; @@ -58,162 +62,6 @@ use respond::actuator::{ }; use std::collections::{HashMap, HashSet}; -/// OTLP instruments for the engine, recorded against the global meter (see -/// [`crate::telemetry`]). When no OTLP endpoint is configured the global meter is a -/// no-op, so these calls cost nothing — the engine is instrumented unconditionally. -/// Counters are cumulative; gauges hold the last pass's snapshot. -struct EngineMetrics { - /// Process passes (one per observed change). - passes: opentelemetry::metrics::Counter, - /// Adjudicator model invocations, by `result` (`ok`/`unavailable`). - model_calls: opentelemetry::metrics::Counter, - /// Mitigations actuated, by `action` (`applied`/`reverted`). - mitigations: opentelemetry::metrics::Counter, - /// Proven chains in the last pass. - chains: opentelemetry::metrics::Gauge, - /// Breach-relevant findings (internet-facing) in the last pass. - breach_paths: opentelemetry::metrics::Gauge, - /// Active mitigations currently in the ledger. - active_mitigations: opentelemetry::metrics::Gauge, - /// Breach-path count by model `verdict` (the current judgement distribution). - verdicts: opentelemetry::metrics::Gauge, - /// Behavioral signals ingested this pass, by `behavior` variant (alert/connection/ - /// secret-read/library-load/file-read/priv-change/exec) — the shadow-bake (JEF-48) - /// view of *what* the behavioral port is seeing, labeled low-cardinality (variant - /// names only, never per-pod). - signals: opentelemetry::metrics::Counter, - /// Signal attribution outcome, by `outcome` (`resolved`/`unresolved`): how many - /// ingested signals the runtime adapter could attribute to a live workload vs drop as - /// an unknown cgroup UID. A sustained `unresolved` means the agent's UIDs aren't - /// matching pod metadata. - attribution: opentelemetry::metrics::Counter, - /// `RuntimeEvents` store cardinality (distinct live observations) as of this pass — - /// a gauge so the TTL'd store's working-set size is observable. - runtime_store: opentelemetry::metrics::Gauge, - /// Corroborations fired this pass: proven breach-relevant chains whose `corroborated` - /// predicate is set (ADR-0009). In shadow this is the countable answer to "would this - /// have promoted?" without any behavior change. - corroborations: opentelemetry::metrics::Counter, - /// Per-pass adjudications that issued a fresh model call (verdict-cache miss). A - /// proper cumulative counter (replaces the prior `verdicts{verdict="judged_this_pass"}` - /// gauge hack) so model-call frequency is rate-able. - judged: opentelemetry::metrics::Counter, - /// Per-pass adjudications served from the verdict cache (cache hit). Cumulative - /// counter, the companion to [`Self::judged`]. - cached: opentelemetry::metrics::Counter, - /// Per-pass cache MISSES the engine declined to send to the model because the entry - /// (or the whole fleet, via the global breaker) was in inconclusive-adjudication - /// backoff (JEF-234). A cumulative counter: a sustained nonzero rate means the model is - /// degraded and the engine is correctly NOT hammering it (the bounding is working). - skipped: opentelemetry::metrics::Counter, - /// Adjudicator model-call latency in milliseconds (histogram), recorded around each - /// fresh `judge` call so the slow CPU model's tail is visible. - model_latency_ms: opentelemetry::metrics::Histogram, - /// Shadow would-have-acted report headline (JEF-143): distinct workloads the engine - /// WOULD have isolated over the default rolling window, as of this pass — the gates- - /// exiting-shadow figure (JEF-50), mirrored to OTLP like the bake counts. A gauge: - /// the current window snapshot, the in-process mirror of the would-have-acted report - /// aggregation. - report_would_act: opentelemetry::metrics::Gauge, - /// Shadow report headline: distinct proven-but-cleared paths the model left alone - /// over the window (the trust half of the diff). - report_left_alone: opentelemetry::metrics::Gauge, - /// Shadow report headline: would-acts whose projected cut was short-lived (likely - /// false positives) — the subset to discount when judging the shadow bake. - report_short_lived: opentelemetry::metrics::Gauge, - /// Shadow report headline: would-acts made during an enrichment-coverage gap (no CVE - /// backing) — the ones to scrutinize first. - report_coverage_gap: opentelemetry::metrics::Gauge, -} - -impl EngineMetrics { - fn new() -> Self { - let m = opentelemetry::global::meter("protector.engine"); - Self { - passes: m - .u64_counter("protector.engine.passes") - .with_description("Engine process passes (one per observed change).") - .build(), - model_calls: m - .u64_counter("protector.engine.model_calls") - .with_description("Adjudicator model invocations by result.") - .build(), - mitigations: m - .u64_counter("protector.engine.mitigations") - .with_description("Mitigations actuated by action.") - .build(), - chains: m - .u64_gauge("protector.engine.chains") - .with_description("Proven chains in the last pass.") - .build(), - breach_paths: m - .u64_gauge("protector.engine.breach_paths") - .with_description("Breach-relevant (internet-facing) findings in the last pass.") - .build(), - active_mitigations: m - .u64_gauge("protector.engine.active_mitigations") - .with_description("Active mitigations in the ledger.") - .build(), - verdicts: m - .u64_gauge("protector.engine.verdicts") - .with_description("Breach paths by model verdict (current distribution).") - .build(), - signals: m - .u64_counter("protector.engine.signals") - .with_description("Behavioral signals ingested by variant.") - .build(), - attribution: m - .u64_counter("protector.engine.attribution") - .with_description("Signal attribution outcome (resolved/unresolved).") - .build(), - runtime_store: m - .u64_gauge("protector.engine.runtime_store") - .with_description("RuntimeEvents store cardinality (live observations).") - .build(), - corroborations: m - .u64_counter("protector.engine.corroborations") - .with_description("Corroborations fired (corroborated breach chains) per pass.") - .build(), - judged: m - .u64_counter("protector.engine.judged") - .with_description("Adjudications that issued a fresh model call (cache miss).") - .build(), - cached: m - .u64_counter("protector.engine.cached") - .with_description("Adjudications served from the verdict cache (cache hit).") - .build(), - skipped: m - .u64_counter("protector.engine.skipped") - .with_description( - "Adjudications skipped (inconclusive-adjudication backoff / breaker open).", - ) - .build(), - model_latency_ms: m - .f64_histogram("protector.engine.model_latency_ms") - .with_description("Adjudicator model-call latency in milliseconds.") - .build(), - report_would_act: m - .u64_gauge("protector.engine.report_would_act") - .with_description( - "Shadow report: workloads that would have been isolated (window).", - ) - .build(), - report_left_alone: m - .u64_gauge("protector.engine.report_left_alone") - .with_description("Shadow report: proven-but-cleared paths left alone (window).") - .build(), - report_short_lived: m - .u64_gauge("protector.engine.report_short_lived") - .with_description("Shadow report: short-lived would-acts (likely false positives).") - .build(), - report_coverage_gap: m - .u64_gauge("protector.engine.report_coverage_gap") - .with_description("Shadow report: would-acts during an enrichment-coverage gap.") - .build(), - } - } -} - /// The engine's stateful processing core. It owns everything that persists across /// observations — the prior graph state, the mitigation ledger, and the applied- /// action log — and exposes one operation, [`Engine::process`], run once per @@ -271,6 +119,9 @@ pub struct Engine { /// end-of-pass re-publish lag. Pruned to present entries each pass (ephemeral workloads, /// removed exposure). verdicts: std::sync::Arc, + /// Per-node agent-liveness (JEF-308), shared with the ingest; classified each pass into the + /// runtime-corroboration coverage the readiness row reads. `None` when no ingest is wired. + agent_liveness: Option>, /// OTLP instruments (no-op when no collector is configured). metrics: EngineMetrics, } @@ -318,10 +169,17 @@ impl Engine { ledger: MitigationLedger::new(), actions: ActionLog::new(), verdicts, + agent_liveness: None, metrics: EngineMetrics::new(), } } + /// Attach the per-node agent-liveness store (JEF-308), read each pass to stamp coverage. + pub fn with_agent_liveness(mut self, l: std::sync::Arc) -> Self { + self.agent_liveness = Some(l); + self + } + /// Attach a durable decision journal (JEF-141) and replay it onto the in-memory /// state, so the findings snapshot, the resolved verdicts, and the reversion log populate /// IMMEDIATELY after a restart — before a fresh (slow CPU) model pass lands. A disabled @@ -542,13 +400,9 @@ impl Engine { // known live verdict, or a journal-restored one). So this single publish already // shows the carried-forward verdict, and when the judging loop below writes a // fresh verdict into the store it is resolved IMMEDIATELY — no end-of-pass - // re-publish is needed to surface it. - self.findings.replace( - chains - .iter() - .map(|c| state::Finding::from_chain(c, &graph)) - .collect(), - ); + // re-publish is needed to surface it. `publish_chains` also stamps each finding's entry + // node (JEF-308) so a latent finding on a blind node can carry its "no live sensor" caveat. + self.findings.publish_chains(&chains, &graph, snapshot); // Snapshot gauges for this pass. self.metrics.chains.record(chains.len() as u64, &[]); @@ -573,6 +427,12 @@ impl Engine { bake.corroborations = corroborations; self.findings.set_bake(bake); + // Runtime-corroboration coverage per node (JEF-308) for the readiness row. + if let Some(liveness) = &self.agent_liveness { + self.findings + .stamp_runtime_coverage(liveness, &snapshot.pods); + } + // Adjudicate (ADR-0013): the model is the JUDGE of every breach-relevant PATH, // always. The deterministic proof winnows to the paths an internet-facing // workload can actually reach (internet → entry → objective); the model then @@ -843,13 +703,8 @@ impl Engine { // `adjudicated`, so the disposition is current. JEF-157: the VERDICT is no longer // what this re-publish is for (it was already written to the shared store the // instant each entry was judged, and the findings snapshot resolves it from there) — - // this only refreshes the structural enrichment of the rows. - self.findings.replace( - chains - .iter() - .map(|c| state::Finding::from_chain(c, &graph)) - .collect(), - ); + // this only refreshes the structural enrichment of the rows (+ re-stamps entry nodes). + self.findings.publish_chains(&chains, &graph, snapshot); if structurally_changed && !chains.is_empty() { tracing::info!(count = chains.len(), "proven chains"); diff --git a/engine/src/engine/observe/adapter/enrich.rs b/engine/src/engine/observe/adapter/enrich.rs index becc1c2..70ab528 100644 --- a/engine/src/engine/observe/adapter/enrich.rs +++ b/engine/src/engine/observe/adapter/enrich.rs @@ -665,6 +665,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::LibraryLoaded { name: name.into() }, }] }) @@ -707,6 +708,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::LibraryLoaded { name: name.into() }, } } @@ -907,6 +909,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::NetworkConnection { peer: peer.into(), internet, @@ -958,6 +961,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::LibraryLoaded { name: "anything.so".into(), }, diff --git a/engine/src/engine/observe/mod.rs b/engine/src/engine/observe/mod.rs index dc0ce08..3c4dfb9 100644 --- a/engine/src/engine/observe/mod.rs +++ b/engine/src/engine/observe/mod.rs @@ -214,7 +214,7 @@ pub(crate) fn report_resource(object: &DynamicObject) -> Option { /// definition rather than a hand-synced duplicate. Re-exported here because the Observer /// and adapters refer to it as `observe::RuntimeObservation`. [`Attribution`] (how an /// observation is attributed to a workload) is re-exported alongside it for the same reason. -pub use protector_behavior::{Attribution, RuntimeObservation}; +pub use protector_behavior::{AgentReport, Attribution, RuntimeObservation}; /// The normalized API secret-read lifted from the apiserver audit log (JEF-269) — the /// corroborating runtime signal the eBPF agent can't observe. Re-exported here because the diff --git a/engine/src/engine/observe/runtime.rs b/engine/src/engine/observe/runtime.rs index 0221bf1..d850b36 100644 --- a/engine/src/engine/observe/runtime.rs +++ b/engine/src/engine/observe/runtime.rs @@ -28,8 +28,9 @@ use tokio::sync::mpsc::Sender; use super::ingest_guard::{ DEFAULT_BURST, DEFAULT_RATE_PER_SEC, IngestToken, RateLimit, bearer_auth, rate_limit, }; -use super::{Attribution, RuntimeObservation}; +use super::{AgentReport, Attribution, RuntimeObservation}; use crate::engine::graph::Behavior; +use crate::engine::state::AgentLivenessStore; /// A time-windowed store of recent runtime observations. Thread-safe so the HTTP /// ingest task and the engine loop can share it. @@ -140,18 +141,20 @@ pub fn parse_falco_event(event: &Value) -> Option { // Falco's wall-clock `time` could be parsed here; until then the adapter // stamps ingest-time. Falco is the low-volume sensor, so the lag is minor. observed_at_ms: None, + node: None, behavior: Behavior::Alert { rule }, }) } -/// Shared state for the ingest handler: the event store and a wake channel. -type IngestState = (Arc, Sender<()>); +/// Shared state for the ingest handlers: the event store, a wake channel, and the per-node +/// agent-liveness store (JEF-308) the `/agent-liveness` beacon feeds. +type IngestState = (Arc, Sender<()>, Arc); /// Receive one Falco alert, record it, and wake the engine. Unparseable or /// non-pod alerts are accepted and ignored (we still 200 so falcosidekick doesn't /// retry-storm). async fn ingest( - State((events, notify)): State, + State((events, notify, _liveness)): State, Json(event): Json, ) -> StatusCode { if let Some(observation) = parse_falco_event(&event) { @@ -172,7 +175,7 @@ async fn ingest( /// connections continuously, so most batches are pure repeats — those refresh TTLs but /// must not churn a process pass, which is the whole point of gating the wake here. async fn ingest_behavior( - State((events, notify)): State, + State((events, notify, _liveness)): State, Json(observations): Json>, ) -> StatusCode { if record_batch(&events, observations) { @@ -181,6 +184,21 @@ async fn ingest_behavior( StatusCode::OK } +/// Receive a batch of per-node [`AgentReport`] liveness beacons (JEF-308) and record them into the +/// liveness store. Unlike a behavior, a beacon does NOT wake the engine — liveness is read at pass +/// time; a beacon only refreshes freshness. A beacon arrives every window even when the node saw +/// nothing, which is exactly what lets a quiet node read HEALTHY-quiet instead of blind. Same body +/// cap / authn / rate limit as the sibling routes; over-cap batches are truncated defensively. +async fn ingest_agent_liveness( + State((_events, _notify, liveness)): State, + Json(reports): Json>, +) -> StatusCode { + for report in reports.into_iter().take(MAX_BATCH) { + liveness.record(report); + } + StatusCode::OK +} + /// Upper bound on observations accepted per `/behavior` batch. Each `record()` is an /// O(n) scan over up to `MAX_EVENTS` entries while the store mutex is held, so an /// oversized batch would do O(batch x MAX_EVENTS) work under the lock — a cheap DoS @@ -219,6 +237,7 @@ pub async fn serve_runtime( addr: SocketAddr, events: Arc, notify: Sender<()>, + liveness: Arc, ) -> anyhow::Result<()> { // App-layer authn (Fix A): require `Authorization: Bearer ` matching a // configured shared secret, rejected 401 BEFORE deserialization. Resolved once at @@ -234,11 +253,13 @@ pub async fn serve_runtime( let mut app = Router::new() .route("/", post(ingest)) .route("/behavior", post(ingest_behavior)) + // The per-node agent-liveness beacon (JEF-308) — signal-flow liveness, not pod-Ready. + .route("/agent-liveness", post(ingest_agent_liveness)) // A real alert/batch is small; cap the body so a client can't OOM the engine // with a giant POST (mirrors the webhook server). The body cap, MAX_EVENTS, and // the per-batch MAX_BATCH all remain in force alongside authn + rate limiting. .layer(DefaultBodyLimit::max(256 * 1024)) - .with_state((events, notify)); + .with_state((events, notify, liveness)); // Rate limit runs on every request, authenticated or not. app = app.layer(axum::middleware::from_fn_with_state(limiter, rate_limit)); @@ -246,7 +267,7 @@ pub async fn serve_runtime( match token { Some(token) => { app = app.layer(axum::middleware::from_fn_with_state(token, bearer_auth)); - tracing::info!(%addr, "runtime-evidence ingest listening (/, /behavior) — bearer-authenticated"); + tracing::info!(%addr, "runtime-evidence ingest listening (/, /behavior, /agent-liveness) — bearer-authenticated"); } None => { tracing::warn!( @@ -278,6 +299,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::Alert { rule: rule.into() }, } } diff --git a/engine/src/engine/reason/adjudicate/tests/group_1.rs b/engine/src/engine/reason/adjudicate/tests/group_1.rs index 8889ee1..9d3e632 100644 --- a/engine/src/engine/reason/adjudicate/tests/group_1.rs +++ b/engine/src/engine/reason/adjudicate/tests/group_1.rs @@ -528,6 +528,7 @@ fn prompt_includes_the_chain_evidence() { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: crate::engine::graph::Behavior::Alert { rule: "Terminal shell in container".into(), }, diff --git a/engine/src/engine/reason/proof/corroborate_tests.rs b/engine/src/engine/reason/proof/corroborate_tests.rs index b869a38..8946bcd 100644 --- a/engine/src/engine/reason/proof/corroborate_tests.rs +++ b/engine/src/engine/reason/proof/corroborate_tests.rs @@ -102,6 +102,7 @@ fn web_entry_with_signal(behavior: Behavior) -> Vec { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior, }], ..Default::default() diff --git a/engine/src/engine/reason/proof/tests.rs b/engine/src/engine/reason/proof/tests.rs index ee9fc6a..e9a08e7 100644 --- a/engine/src/engine/reason/proof/tests.rs +++ b/engine/src/engine/reason/proof/tests.rs @@ -478,6 +478,7 @@ fn runtime_signal_completes_the_action_bar() { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: crate::engine::graph::Behavior::Alert { rule: "Outbound connection to C2".into(), }, @@ -760,6 +761,7 @@ fn secret_read_signal_corroborates_credential_chain_end_to_end() { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::SecretRead { secret: "session-key".into(), source: crate::engine::graph::SecretReadSource::Mounted, @@ -830,6 +832,7 @@ fn library_load_signal_corroborates_through_foothold_end_to_end() { attribution: Attribution::by_namespaced_name("app", "web"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::LibraryLoaded { name: "libssl.so.1.1".into(), }, @@ -898,6 +901,7 @@ fn library_load_does_not_corroborate_without_foothold() { attribution: Attribution::by_namespaced_name("app", "app"), source: None, observed_at_ms: None, + node: None, behavior: Behavior::LibraryLoaded { name: "libssl.so.1.1".into(), }, diff --git a/engine/src/engine/respond/tests.rs b/engine/src/engine/respond/tests.rs index 7a0bbc8..b8d386c 100644 --- a/engine/src/engine/respond/tests.rs +++ b/engine/src/engine/respond/tests.rs @@ -530,6 +530,7 @@ fn internal_active_snapshot(with_alert: bool) -> Snapshot { attribution: Attribution::by_namespaced_name("app", "watcher"), source: Some("falco".into()), observed_at_ms: None, + node: None, behavior: Behavior::Alert { rule: "Terminal shell in container".into(), }, diff --git a/engine/src/engine/run_loop.rs b/engine/src/engine/run_loop.rs index 83c893c..966ee98 100644 --- a/engine/src/engine/run_loop.rs +++ b/engine/src/engine/run_loop.rs @@ -290,6 +290,13 @@ pub async fn run_watch( // Diagnostic judgement log: the full prompt + raw reply + verdict per judgement, // written by the adjudicator for later inspection. let journal = std::sync::Arc::new(state::JudgementLog::new()); + // Per-node agent-liveness (JEF-308): the TTL'd store the `/agent-liveness` beacon feeds and + // the engine reads each pass to classify runtime-corroboration coverage per node. Same 300s + // freshness window as the runtime feed — a node whose agent stopped beaconing goes stale and + // reads blind. Shared (`Arc`) between the ingest task and the engine loop. + let agent_liveness = std::sync::Arc::new(state::AgentLivenessStore::new( + std::time::Duration::from_secs(300), + )); // The durable decision journal (JEF-141): reload pre-restart decisions onto the // in-memory state so the output state isn't blank while the caches + CPU model warm. // Unset/unwritable `PROTECTOR_ENGINE_JOURNAL_PATH` ⇒ disabled (in-memory only, no @@ -305,7 +312,10 @@ pub async fn run_watch( // The one sanctioned outbound path (JEF-144, ADR-0018): operator-configured via // `PROTECTOR_ENGINE_NOTIFY_URL`, off (zero outbound calls) when unset, redacted by // default. - .with_notifier(notify::BreachNotifier::from_env()); + .with_notifier(notify::BreachNotifier::from_env()) + // The per-node agent-liveness store (JEF-308): read each pass to stamp the runtime-corroboration + // coverage into the findings handle the readiness aggregation reads. + .with_agent_liveness(agent_liveness.clone()); // Repopulate the webhook's admission-decision ring from the durable journal on boot // (JEF-237), so the admission-decision log isn't blank after a restart — parallel to how @@ -433,8 +443,11 @@ pub async fn run_watch( let (runtime_tx, mut runtime_rx) = tokio::sync::mpsc::channel::<()>(64); if let Some(addr) = runtime_addr { let events = runtime_events.clone(); + let liveness = agent_liveness.clone(); tokio::spawn(async move { - if let Err(error) = observe::runtime::serve_runtime(addr, events, runtime_tx).await { + if let Err(error) = + observe::runtime::serve_runtime(addr, events, runtime_tx, liveness).await + { tracing::error!(%error, "runtime-evidence ingest stopped"); } }); diff --git a/engine/src/engine/state/agent_liveness.rs b/engine/src/engine/state/agent_liveness.rs new file mode 100644 index 0000000..255a9a3 --- /dev/null +++ b/engine/src/engine/state/agent_liveness.rs @@ -0,0 +1,331 @@ +//! Per-node **agent-liveness** (JEF-308): the live [`AgentLivenessStore`] the ingest feeds from +//! the agent's [`AgentReport`] beacons, the **expected-node set** derived from the in-cluster pod +//! informer, and the pure [`derive_runtime_coverage`] that classifies each expected node +//! healthy / degraded / blind — the honest data behind the collapsed "Runtime corroboration" +//! readiness row. +//! +//! The design point is **honesty**: a per-node "blind on node X" view must never invent data, and +//! it must never read missing data as reassuring. +//! +//! * **Signal-flow, not pod-Ready.** A Ready agent whose eBPF probes failed to attach is still +//! BLIND (the exact Falco failure mode). The agent reports `probes_loaded`, so an +//! agent-up-but-`probes_loaded==0` node reads blind, not healthy. +//! * **Quiet ≠ blind.** A node reporting with `signals_emitted == 0` and its probes loaded is +//! HEALTHY-quiet — a quiet cluster is not a down sensor. +//! * **Out-of-scope ≠ blind.** The expected-node set is exactly where the scheduler placed the +//! agent DaemonSet (it already honoured the agent's nodeSelector/tolerations — today +//! `kubernetes.io/arch: arm64`, JEF-295). A node the agent is NOT scheduled on has no agent +//! pod, so it is simply absent from the expected set — out-of-scope, never blind. +//! +//! Zero-egress: liveness is derived from the observations/beacons the agent already POSTs plus the +//! in-cluster informer (JEF-131) — the agent's OTLP/metrics endpoint is never consumed. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use k8s_openapi::api::core::v1::Pod; +use protector_behavior::AgentReport; + +/// The label the agent DaemonSet's pods carry (chart helper `protector.agentLabels`). We pick the +/// agent's OWN pods out of the pod informer by it, so the expected-node set is exactly the nodes +/// the scheduler placed the agent on — respecting its nodeSelector/tolerations by construction, +/// with no need to re-implement selector matching (which would drift from the chart). +const AGENT_COMPONENT_LABEL: &str = "app.kubernetes.io/component"; +/// The value of [`AGENT_COMPONENT_LABEL`] on an agent pod. +const AGENT_COMPONENT_VALUE: &str = "agent"; + +/// One node's most-recent liveness report, held with its ingest time for the TTL. +#[derive(Debug, Clone, Copy)] +struct NodeReport { + at: Instant, + probes_loaded: u32, + probes_total: u32, + signals: u64, +} + +/// A node's current liveness facts, as read out of the store (TTL already applied). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LiveNode { + pub probes_loaded: u32, + pub probes_total: u32, + pub signals: u64, +} + +/// The live per-node agent-liveness store: the last [`AgentReport`] seen from each node, kept only +/// while fresh (a report older than the TTL is treated as "not reporting" → blind). Thread-safe so +/// the HTTP ingest task and the engine loop share one `Arc`. Latest report per node wins. +pub struct AgentLivenessStore { + inner: Mutex>, + ttl: Duration, +} + +impl AgentLivenessStore { + /// Hard cap on distinct nodes retained, independent of the TTL: a bearer-holding client could + /// otherwise flood the ingest with distinct (attacker-chosen) node names and grow this map + /// without bound within one TTL window. A real cluster has far fewer nodes than this; at the + /// cap a new node evicts the stalest entry, so genuine reporters are never starved. + const MAX_NODES: usize = 4096; + + /// A store whose reports go stale after `ttl`. A node whose last beacon is older than `ttl` + /// reads as not-reporting (blind) — freshness is a first-class correctness concern (ADR-0002): + /// an agent that stopped beaconing must not keep reading healthy off a stale report. + pub fn new(ttl: Duration) -> Self { + Self { + inner: Mutex::new(BTreeMap::new()), + ttl, + } + } + + /// Record a beacon as of now, replacing any prior report for the same node. + pub fn record(&self, report: AgentReport) { + self.record_at(Instant::now(), report); + } + + /// Record a beacon as of `now` — the deterministic seam the tests drive. + pub fn record_at(&self, now: Instant, report: AgentReport) { + let mut inner = self.inner.lock().expect("agent-liveness mutex poisoned"); + // Prune stale entries first, then bound the map: if this is a NEW node and we're at the + // cap, evict the stalest entry so a flood of distinct node names can't grow it unbounded. + inner.retain(|_, r| now.duration_since(r.at) < self.ttl); + if inner.len() >= Self::MAX_NODES + && !inner.contains_key(&report.node) + && let Some(oldest) = inner + .iter() + .min_by_key(|(_, r)| r.at) + .map(|(node, _)| node.clone()) + { + inner.remove(&oldest); + } + inner.insert( + report.node, + NodeReport { + at: now, + probes_loaded: report.probes_loaded, + probes_total: report.probes_total, + signals: report.signals_emitted, + }, + ); + } + + /// The per-node facts still within the TTL as of now. + pub fn snapshot(&self) -> BTreeMap { + self.snapshot_at(Instant::now()) + } + + /// The per-node facts still within the TTL as of `now`, pruning stale reports in place so a + /// node that stopped beaconing drops out and reads blind on the next pass. + pub fn snapshot_at(&self, now: Instant) -> BTreeMap { + let mut inner = self.inner.lock().expect("agent-liveness mutex poisoned"); + inner.retain(|_, r| now.duration_since(r.at) < self.ttl); + inner + .iter() + .map(|(node, r)| { + ( + node.clone(), + LiveNode { + probes_loaded: r.probes_loaded, + probes_total: r.probes_total, + signals: r.signals, + }, + ) + }) + .collect() + } +} + +/// The **expected-node set** for the agent (JEF-308): the distinct nodes the agent DaemonSet's own +/// pods are scheduled on, read straight from the pod informer. Because the scheduler already +/// honoured the agent's nodeSelector/tolerations when it placed those pods, this set IS the +/// "should be running the agent" set — a node the agent isn't scheduled on simply has no agent pod +/// here, so it's out-of-scope, never blind. Pure over its input. +pub fn expected_agent_nodes(pods: &[Pod]) -> BTreeSet { + pods.iter() + .filter(|p| is_agent_pod(p)) + .filter_map(|p| p.spec.as_ref().and_then(|s| s.node_name.clone())) + .filter(|n| !n.is_empty()) + .collect() +} + +/// Whether a pod is one of the agent DaemonSet's pods (by the component label the chart sets). +fn is_agent_pod(pod: &Pod) -> bool { + pod.metadata + .labels + .as_ref() + .and_then(|l| l.get(AGENT_COMPONENT_LABEL)) + .map(|v| v == AGENT_COMPONENT_VALUE) + .unwrap_or(false) +} + +/// Why a node reads blind — carried so the UX can name the failure honestly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlindReason { + /// No fresh beacon from this expected node — the agent is gone / crashed / never came up. + NotReporting, + /// The agent reported but attached ZERO probes — Ready but blind (the Falco failure mode). + ProbesFailed, +} + +/// One node's classified runtime-corroboration state this pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeState { + /// Reporting with its probes loaded — contributing corroboration. `signals` may be 0 + /// (HEALTHY-quiet: a quiet node is not a down sensor). + Healthy { signals: u64 }, + /// Reporting but only SOME probes attached — partial (degraded) coverage. + DegradedProbes { loaded: u32, total: u32 }, + /// No live corroboration from this expected node — see [`BlindReason`]. + Blind { reason: BlindReason }, + /// Reporting from a node NOT in the expected set (agent running where it isn't scheduled) — + /// out-of-scope, explicitly not blind. + OutOfScope, +} + +impl NodeState { + /// Whether this node is blind (an expected node with no live corroboration). + pub fn is_blind(self) -> bool { + matches!(self, NodeState::Blind { .. }) + } +} + +/// One node's coverage row (node name + its classified state). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeCoverage { + /// The node name — UNTRUSTED-adjacent at render (escape it, never `PreEscaped`). + pub node: String, + pub state: NodeState, +} + +/// The per-pass runtime-corroboration coverage: one [`NodeCoverage`] per expected node (plus any +/// out-of-scope reporters), the honest input the readiness row + strip chip read. Purely derived — +/// it holds no rendering and makes no decision (ADR-0016). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RuntimeCoverage { + pub nodes: Vec, +} + +impl RuntimeCoverage { + /// How many nodes are IN SCOPE (expected) — out-of-scope reporters don't count. + pub fn expected_count(&self) -> usize { + self.nodes + .iter() + .filter(|n| !matches!(n.state, NodeState::OutOfScope)) + .count() + } + + /// The blind expected nodes, by name — what the "degraded (blind on N)" ladder rung names. + pub fn blind_nodes(&self) -> Vec<&str> { + self.nodes + .iter() + .filter(|n| n.state.is_blind()) + .map(|n| n.node.as_str()) + .collect() + } + + /// The expected nodes reporting only partial probes, by name. + pub fn degraded_nodes(&self) -> Vec<&str> { + self.nodes + .iter() + .filter(|n| matches!(n.state, NodeState::DegradedProbes { .. })) + .map(|n| n.node.as_str()) + .collect() + } + + /// How many expected nodes are fully healthy (probes loaded; quiet counts). + pub fn healthy_count(&self) -> usize { + self.nodes + .iter() + .filter(|n| matches!(n.state, NodeState::Healthy { .. })) + .count() + } + + /// Total signals the agent emitted this pass across healthy nodes. + pub fn agent_signals(&self) -> u64 { + self.nodes + .iter() + .filter_map(|n| match n.state { + NodeState::Healthy { signals } => Some(signals), + _ => None, + }) + .sum() + } + + /// Whether every expected node is fully healthy (and there is at least one expected node). + pub fn all_healthy(&self) -> bool { + self.expected_count() > 0 + && self + .nodes + .iter() + .filter(|n| !matches!(n.state, NodeState::OutOfScope)) + .all(|n| matches!(n.state, NodeState::Healthy { .. })) + } + + /// The set of blind node names — the finding-caveat lookup ("no live sensor on this node"). + pub fn blind_node_set(&self) -> HashSet { + self.nodes + .iter() + .filter(|n| n.state.is_blind()) + .map(|n| n.node.clone()) + .collect() + } +} + +/// Classify the runtime-corroboration coverage from the expected-node set and the live liveness +/// snapshot (JEF-308). Pure and total — the tested honesty core: +/// +/// * an expected node with a fresh report + probes loaded ⇒ [`Healthy`] (quiet or not); +/// * an expected node reporting `probes_loaded == 0` ⇒ [`Blind`]`(ProbesFailed)` — Ready-but-blind; +/// * an expected node with only some probes ⇒ [`DegradedProbes`]; +/// * an expected node with no fresh report ⇒ [`Blind`]`(NotReporting)`; +/// * a node reporting that is NOT expected ⇒ [`OutOfScope`] (never blind). +/// +/// [`Healthy`]: NodeState::Healthy +/// [`Blind`]: NodeState::Blind +/// [`DegradedProbes`]: NodeState::DegradedProbes +/// [`OutOfScope`]: NodeState::OutOfScope +pub fn derive_runtime_coverage( + expected: &BTreeSet, + live: &BTreeMap, +) -> RuntimeCoverage { + let mut nodes: Vec = Vec::new(); + + // Every expected node, classified against its (possibly absent) live report. + for node in expected { + let state = match live.get(node) { + None => NodeState::Blind { + reason: BlindReason::NotReporting, + }, + Some(l) if l.probes_loaded == 0 => NodeState::Blind { + reason: BlindReason::ProbesFailed, + }, + Some(l) if l.probes_total > 0 && l.probes_loaded < l.probes_total => { + NodeState::DegradedProbes { + loaded: l.probes_loaded, + total: l.probes_total, + } + } + // Probes loaded (fully, or the build declares no denominator) — healthy, quiet or not. + Some(l) => NodeState::Healthy { signals: l.signals }, + }; + nodes.push(NodeCoverage { + node: node.clone(), + state, + }); + } + + // A node reporting that isn't in the expected set: out-of-scope, explicitly not blind. + for node in live.keys() { + if !expected.contains(node) { + nodes.push(NodeCoverage { + node: node.clone(), + state: NodeState::OutOfScope, + }); + } + } + + nodes.sort_by(|a, b| a.node.cmp(&b.node)); + RuntimeCoverage { nodes } +} + +#[cfg(test)] +mod tests; diff --git a/engine/src/engine/state/agent_liveness/tests.rs b/engine/src/engine/state/agent_liveness/tests.rs new file mode 100644 index 0000000..9b167f2 --- /dev/null +++ b/engine/src/engine/state/agent_liveness/tests.rs @@ -0,0 +1,225 @@ +//! Tests for the per-node agent-liveness honesty core (JEF-308): the expected-node set from the +//! informer, and the healthy / degraded / blind / out-of-scope / quiet≠blind classification. + +use std::time::{Duration, Instant}; + +use k8s_openapi::api::core::v1::Pod; +use protector_behavior::AgentReport; +use serde_json::json; + +use super::*; + +/// An agent DaemonSet pod on `node`, carrying the component label the chart sets. +fn agent_pod(name: &str, node: &str) -> Pod { + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Pod", + "metadata": { + "name": name, "namespace": "protector", + "labels": {"app.kubernetes.io/component": "agent"} + }, + "spec": {"nodeName": node, "containers": [{"name": "agent", "image": "agent:1"}]} + })) + .unwrap() +} + +/// A NON-agent workload pod on `node` (no agent component label). +fn app_pod(name: &str, node: &str) -> Pod { + serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": name, "namespace": "app", "labels": {"app": "web"}}, + "spec": {"nodeName": node, "containers": [{"name": "c", "image": "web:1"}]} + })) + .unwrap() +} + +fn report(node: &str, probes_loaded: u32, probes_total: u32, signals: u64) -> AgentReport { + AgentReport { + node: node.into(), + probes_loaded, + probes_total, + signals_emitted: signals, + observed_at_ms: None, + } +} + +#[test] +fn expected_set_is_where_the_agent_is_scheduled_only() { + // The scheduler already honoured the agent's nodeSelector/tolerations (JEF-295 arm64), so the + // expected set is exactly the agent pods' nodes. A node running only a non-agent workload + // (an amd64 node the agent isn't scheduled on) is NOT expected — out-of-scope, not blind. + let pods = vec![ + agent_pod("agent-a", "node-a"), + agent_pod("agent-b", "node-b"), + app_pod("web-1", "node-c"), + ]; + let expected = expected_agent_nodes(&pods); + assert!(expected.contains("node-a")); + assert!(expected.contains("node-b")); + assert!( + !expected.contains("node-c"), + "a node the agent isn't scheduled on is out-of-scope, never blind" + ); + assert_eq!(expected.len(), 2); +} + +#[test] +fn reporting_node_with_probes_loaded_is_healthy() { + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 6, 6, 9)); + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0)); + assert_eq!(cov.expected_count(), 1); + assert_eq!(cov.healthy_count(), 1); + assert!(cov.blind_nodes().is_empty()); + assert!(cov.all_healthy()); + assert_eq!(cov.agent_signals(), 9); + assert_eq!(cov.nodes[0].state, NodeState::Healthy { signals: 9 }); +} + +#[test] +fn quiet_node_is_healthy_not_blind() { + // The load-bearing honesty case: a node reporting with ZERO signals but its probes loaded is + // HEALTHY-quiet — a quiet cluster must never read as a down sensor. + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 6, 6, 0)); + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0)); + assert!(cov.blind_nodes().is_empty(), "quiet ≠ blind"); + assert!(cov.all_healthy()); + assert_eq!(cov.nodes[0].state, NodeState::Healthy { signals: 0 }); +} + +#[test] +fn expected_node_not_reporting_is_blind_and_named() { + // node-b should run the agent but no beacon arrived — blind, and named so the UX can say which. + let expected: BTreeSet = ["node-a".into(), "node-b".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 6, 6, 3)); + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0)); + assert_eq!(cov.blind_nodes(), vec!["node-b"]); + assert!(!cov.all_healthy()); + let b = cov.nodes.iter().find(|n| n.node == "node-b").unwrap(); + assert_eq!( + b.state, + NodeState::Blind { + reason: BlindReason::NotReporting + } + ); +} + +#[test] +fn probes_failed_to_load_is_blind_despite_reporting() { + // Ready but blind: the agent reports but attached ZERO probes — the exact Falco failure mode. + // Pod-Ready would read healthy; signal-flow reads it blind. + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 0, 6, 0)); + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0)); + assert_eq!(cov.blind_nodes(), vec!["node-a"]); + assert_eq!( + cov.nodes[0].state, + NodeState::Blind { + reason: BlindReason::ProbesFailed + } + ); + assert!(cov.blind_node_set().contains("node-a")); +} + +#[test] +fn partial_probes_are_degraded_not_blind() { + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 4, 6, 2)); + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0)); + assert!(cov.blind_nodes().is_empty()); + assert_eq!(cov.degraded_nodes(), vec!["node-a"]); + assert_eq!( + cov.nodes[0].state, + NodeState::DegradedProbes { + loaded: 4, + total: 6 + } + ); +} + +#[test] +fn a_stale_report_prunes_and_reads_blind() { + // A node that stopped beaconing must not keep reading healthy off a stale report — past the + // TTL it prunes out and reads blind (freshness is correctness, ADR-0002). + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 6, 6, 5)); + + // Within the window: healthy. + let fresh = + derive_runtime_coverage(&expected, &store.snapshot_at(t0 + Duration::from_secs(60))); + assert!(fresh.all_healthy()); + // Past the TTL: pruned → blind (not reporting). + let stale = + derive_runtime_coverage(&expected, &store.snapshot_at(t0 + Duration::from_secs(121))); + assert_eq!(stale.blind_nodes(), vec!["node-a"]); +} + +#[test] +fn a_reporting_non_expected_node_is_out_of_scope_not_blind() { + // A beacon from a node NOT in the expected set (agent running where it isn't scheduled) reads + // out-of-scope — never blind, and it doesn't count toward the expected total. + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 6, 6, 1)); + store.record_at(t0, report("node-x", 6, 6, 1)); + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0)); + assert_eq!( + cov.expected_count(), + 1, + "out-of-scope doesn't inflate the expected total" + ); + assert!(cov.blind_nodes().is_empty()); + let x = cov.nodes.iter().find(|n| n.node == "node-x").unwrap(); + assert_eq!(x.state, NodeState::OutOfScope); +} + +#[test] +fn the_store_is_bounded_against_a_distinct_node_flood() { + // A bearer-holding client can't grow the store without bound by flooding distinct node names: + // at the cap a new node evicts the stalest entry. + let store = AgentLivenessStore::new(Duration::from_secs(300)); + let t0 = Instant::now(); + for i in 0..(AgentLivenessStore::MAX_NODES + 100) { + store.record_at( + t0 + Duration::from_millis(i as u64), + report(&format!("n{i}"), 6, 6, 0), + ); + } + assert_eq!( + store.snapshot_at(t0 + Duration::from_secs(1)).len(), + AgentLivenessStore::MAX_NODES + ); +} + +#[test] +fn latest_report_per_node_wins() { + // A newer beacon supersedes an older one for the same node (blind → healthy on recovery). + let expected: BTreeSet = ["node-a".into()].into_iter().collect(); + let store = AgentLivenessStore::new(Duration::from_secs(120)); + let t0 = Instant::now(); + store.record_at(t0, report("node-a", 0, 6, 0)); // blind + store.record_at(t0 + Duration::from_secs(1), report("node-a", 6, 6, 4)); // recovered + + let cov = derive_runtime_coverage(&expected, &store.snapshot_at(t0 + Duration::from_secs(1))); + assert!(cov.all_healthy()); + assert_eq!(cov.agent_signals(), 4); +} diff --git a/engine/src/engine/state/findings.rs b/engine/src/engine/state/findings.rs index e1be65f..c72bdd8 100644 --- a/engine/src/engine/state/findings.rs +++ b/engine/src/engine/state/findings.rs @@ -9,10 +9,16 @@ use std::sync::{Arc, Mutex}; use std::time::{Instant, SystemTime}; +use k8s_openapi::api::core::v1::Pod; + use crate::engine::graph::SecurityGraph; +use crate::engine::observe::Snapshot; use crate::engine::reason::adjudicate::Verdict; use crate::engine::reason::proof::ProvenChain; +use super::agent_liveness::{ + AgentLivenessStore, RuntimeCoverage, derive_runtime_coverage, expected_agent_nodes, +}; use super::evidence::EntryEvidence; use super::recency::RecencyInfo; use super::verdict_store::{BakeStats, ModelHealth, ReadinessConfig, VerdictStore}; @@ -72,6 +78,12 @@ pub struct Finding { /// `None` on a row published before any recency update. Pure presentation metadata /// (ADR-0016). pub recency: Option, + /// The Kubernetes node the entry workload runs on (JEF-308), stamped by the engine from the + /// snapshot when it builds the pass's findings. `None` when the entry isn't a single pod (a + /// multi-replica workload or a non-pod entry) or its node isn't known. Used ONLY to add the + /// blind-node caveat: a latent/propose-only finding on a node with no live sensor must not + /// render as reassuringly calm — absence of a signal there is not evidence of safety. + pub node: Option, } /// One hop of a proven chain: `from -[relation]-> to`, with the **full** node keys @@ -132,8 +144,18 @@ impl Finding { }, paths_truncated: chain.paths_truncated, recency: None, + // Stamped by the engine after construction (it has the snapshot); the chain/graph + // alone don't carry the entry's node. + node: None, } } + + /// Stamp the entry's node (JEF-308) — builder-style, called by the engine when it has the + /// snapshot to resolve the entry pod's `spec.nodeName`. + pub fn with_node(mut self, node: Option) -> Self { + self.node = node; + self + } } /// The one disposition that routes to the remediations set: a reversible network @@ -214,6 +236,11 @@ pub struct Findings { /// adjudication outcome — `0`/`1`/`2` per [`ModelHealth::as_u8`]. Cheap: no extra model /// call, just the result of the call the engine already makes. model_health: std::sync::atomic::AtomicU8, + /// The per-node runtime-corroboration coverage (JEF-308), replaced each pass alongside the + /// findings rows: the expected-node set classified healthy/degraded/blind against the live + /// agent-liveness beacons. The readiness aggregation reads it to build the collapsed + /// "Runtime corroboration" row. Defaults to empty (no expected nodes) until the first pass. + runtime_coverage: Mutex, } impl Findings { @@ -233,6 +260,34 @@ impl Findings { *self.rows.lock().expect("findings mutex poisoned") = findings; } + /// Build and publish this pass's findings from the proven chains, stamping each finding's entry + /// node (JEF-308) from the snapshot so a latent finding on a blind node can carry its "no live + /// sensor here" caveat. Keeps the engine's `process` free of the per-finding node resolution. + pub fn publish_chains( + &self, + chains: &[ProvenChain], + graph: &SecurityGraph, + snapshot: &Snapshot, + ) { + self.replace( + chains + .iter() + .map(|c| { + Finding::from_chain(c, graph).with_node(entry_node(&snapshot.pods, &c.entry.0)) + }) + .collect(), + ); + } + + /// Classify the expected-node set against the live agent-liveness beacons (JEF-308) and stamp + /// the resulting runtime-corroboration coverage. The expected set is exactly the agent + /// DaemonSet's pods in `pods` — the scheduler already honoured the agent's nodeSelector/ + /// tolerations, so a node the agent isn't scheduled on is out-of-scope, never blind. + pub fn stamp_runtime_coverage(&self, liveness: &AgentLivenessStore, pods: &[Pod]) { + let expected = expected_agent_nodes(pods); + self.set_runtime_coverage(derive_runtime_coverage(&expected, &liveness.snapshot())); + } + /// The current findings, each with its verdict resolved from the shared verdict /// store (JEF-157) at read time. The published rows carry no verdict of their own; /// the verdict is looked up per entry here, so a verdict the engine just wrote is @@ -316,6 +371,44 @@ impl Findings { pub fn model_health(&self) -> ModelHealth { ModelHealth::from_u8(self.model_health.load(std::sync::atomic::Ordering::Relaxed)) } + + /// Replace the per-node runtime-corroboration coverage (JEF-308) with this pass's classification. + pub fn set_runtime_coverage(&self, coverage: RuntimeCoverage) { + *self + .runtime_coverage + .lock() + .expect("runtime-coverage mutex poisoned") = coverage; + } + + /// The most recent runtime-corroboration coverage. Defaults to empty (no expected nodes) + /// until the first pass stamps one. + pub fn runtime_coverage(&self) -> RuntimeCoverage { + self.runtime_coverage + .lock() + .expect("runtime-coverage mutex poisoned") + .clone() + } +} + +/// Resolve an entry key's node (JEF-308). Only a single **Pod** entry +/// (`workload//Pod/`) maps to one node — its `spec.nodeName`. A controller workload +/// (a Deployment / DaemonSet entry, whose replicas may span nodes) or a non-pod entry stays +/// node-unattributed (`None`), so the blind-node caveat is never applied to an ambiguous row. +pub(crate) fn entry_node(pods: &[Pod], entry: &str) -> Option { + let parts: Vec<&str> = entry.split('/').collect(); + let [prefix, ns, kind, name] = parts.as_slice() else { + return None; + }; + if *prefix != "workload" || *kind != "Pod" { + return None; + } + pods.iter() + .find(|p| { + p.metadata.namespace.as_deref() == Some(*ns) + && p.metadata.name.as_deref() == Some(*name) + }) + .and_then(|p| p.spec.as_ref().and_then(|s| s.node_name.clone())) + .filter(|n| !n.is_empty()) } #[cfg(test)] @@ -326,6 +419,26 @@ mod tests { use crate::engine::reason::proof::prove; use serde_json::json; + #[test] + fn entry_node_resolves_a_pod_entry_and_none_for_ambiguous_entries() { + // JEF-308: a single-Pod entry resolves to its node; a controller / non-pod entry doesn't. + let pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": "web-1", "namespace": "app"}, + "spec": {"nodeName": "node-a", "containers": [{"name": "c", "image": "web:1"}]} + })) + .unwrap(); + let pods = vec![pod]; + assert_eq!( + entry_node(&pods, "workload/app/Pod/web-1"), + Some("node-a".to_string()) + ); + // A Deployment entry (replicas may span nodes) is node-unattributed. + assert_eq!(entry_node(&pods, "workload/app/Deployment/web"), None); + // An unknown pod resolves to nothing (never guessed). + assert_eq!(entry_node(&pods, "workload/app/Pod/missing"), None); + } + /// A direct breach chain: an internet-facing pod that itself mounts the secret. Its /// only cut is subtractive, so the default containment quarantines the entry — and the /// dashboard disposition must say so, distinct from an edge-cut or a durable-fix PR. @@ -392,6 +505,7 @@ mod tests { attribution: Attribution::by_namespaced_name("app", "watcher"), source: Some("falco".into()), observed_at_ms: None, + node: None, behavior: Behavior::Alert { rule: "Terminal shell in container".into(), }, diff --git a/engine/src/engine/state/mod.rs b/engine/src/engine/state/mod.rs index 1c7b02d..7103e37 100644 --- a/engine/src/engine/state/mod.rs +++ b/engine/src/engine/state/mod.rs @@ -11,6 +11,7 @@ //! verbatim and is the consumer's responsibility to escape wherever it is later spliced into a //! sink (the zero-egress invariant: this state never leaves the cluster). +mod agent_liveness; mod evidence; mod findings; mod judgement; @@ -21,10 +22,14 @@ mod reversion; mod signing_baseline; mod verdict_store; +pub use agent_liveness::{ + AgentLivenessStore, BlindReason, NodeCoverage, NodeState, RuntimeCoverage, + derive_runtime_coverage, expected_agent_nodes, +}; pub use evidence::{CveEvidence, EntryEvidence, FindingEvidence}; pub use findings::{Finding, Findings, PathStep}; pub use judgement::{Judgement, JudgementLog}; -pub use readiness::{InputState, Readiness, ReadinessRow}; +pub use readiness::{InputState, NodeCoverageRow, NodeCoverageState, Readiness, ReadinessRow}; // The dashboard view_model (ADR-0019) derives the live readiness snapshot from the engine's // config + per-pass health, the same pure aggregation the OTLP mirror reads. pub(crate) use readiness::derive_readiness; diff --git a/engine/src/engine/state/readiness.rs b/engine/src/engine/state/readiness.rs index ab53c2a..45256e7 100644 --- a/engine/src/engine/state/readiness.rs +++ b/engine/src/engine/state/readiness.rs @@ -10,6 +10,7 @@ use std::time::SystemTime; use serde::Serialize; +use super::agent_liveness::{BlindReason, NodeState, RuntimeCoverage}; use super::verdict_store::{BakeStats, ModelHealth, ReadinessConfig}; use crate::engine::signing_trust::TUF_STALE_AFTER_SECS; @@ -53,6 +54,39 @@ pub struct ReadinessRow { /// Whether this input being absent WEAKENS the model's decision (the enrichment / /// adjudication inputs — ADR-0016). pub weakens_decisions: bool, + /// The per-node runtime-corroboration breakdown (JEF-308) — populated ONLY for the + /// `runtime-corroboration` row, empty for every other input. Rendered as a server-side + /// `
` inside `
` (no JS) so an operator can see exactly which node is blind. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub nodes: Vec, +} + +/// One node's line in the runtime-corroboration per-node breakdown (JEF-308) — the node name, +/// its honest state, and a short live detail. Node names are UNTRUSTED-adjacent: the render +/// escapes them (maud default), never `PreEscaped`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct NodeCoverageRow { + /// The node name (untrusted-adjacent at render). + pub node: String, + /// The node's honest liveness state. + pub state: NodeCoverageState, + /// A short live detail — signal count, "quiet", probe fraction, or the blind reason. + pub detail: String, +} + +/// One expected node's honest liveness reading (JEF-308) — the per-node mirror of the coarse +/// [`InputState`], kept distinct so "quiet" and "blind" never collapse into one word. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum NodeCoverageState { + /// Reporting with probes loaded — contributing corroboration (signals may be 0 = quiet). + Healthy, + /// Reporting but only some probes attached — partial coverage. + Degraded, + /// No live corroboration on this expected node — the agent is down or its probes failed. + Blind, + /// A node reporting that the agent isn't scheduled on — out-of-scope, explicitly not blind. + OutOfScope, } /// The whole readiness snapshot (JEF-160): every decision input's LIVE state plus the @@ -112,19 +146,14 @@ pub(crate) fn derive_readiness( model_health: ModelHealth, bake: &BakeStats, last_pass: Option, + runtime: &RuntimeCoverage, ) -> Readiness { let warming_up = last_pass.is_none(); - // The behavioral split (JEF-48 variant labels): Falco arrives as the `alert` variant; - // every other variant is an eBPF-agent signal. We report each feed's "signals last - // pass" from the per-variant counts the bake already holds. + // The behavioral split (JEF-48 variant labels): Falco arrives as the `alert` variant. During + // the Falco→agent cutover (F5..F8) it may still feed, so the collapsed row tolerates it as the + // "both-sources" ladder rung — but the row is now agent-SOURCED (per-node, signal-flow). let falco_signals: u64 = bake.signals_by_variant.get("alert").copied().unwrap_or(0); - let ebpf_signals: u64 = bake - .signals_by_variant - .iter() - .filter(|(variant, _)| variant.as_str() != "alert") - .map(|(_, n)| n) - .sum(); // The model is "judging" — giving live verdicts a consumer can lean on — only when it // is attached AND its last fresh call was decisive. A timeout, a cold start, or no model @@ -159,11 +188,12 @@ pub(crate) fn derive_readiness( let kev_state = present_if(config.kev_count > 0); let epss_state = present_if(config.epss_count > 0); - // A behavioral feed is Present iff it delivered >=1 signal this pass, else Absent. (A - // genuinely quiet cluster reads as Absent for the pass — the "signals last pass" detail - // and the cold-start note keep that honest rather than alarming.) - let falco_state = present_if(falco_signals > 0); - let ebpf_state = present_if(ebpf_signals > 0); + // Runtime corroboration (JEF-308): ONE agent-sourced, per-node row replacing the old + // `falco` + `ebpf-agent` split. The honesty ladder — healthy / degraded (blind on N, + // named) / blind (no live signal) / both-sources (Falco still feeding during cutover) — + // and the per-node breakdown come from the derived coverage + this pass's Falco volume. + let (runtime_state, runtime_detail, runtime_nodes) = + runtime_corroboration_row(runtime, falco_signals); let journal_state = present_if(config.journal_durable); @@ -181,6 +211,7 @@ pub(crate) fn derive_readiness( enable: "PROTECTOR_ENGINE_MODEL", detail: model_detail, weakens_decisions: true, + nodes: Vec::new(), }, ReadinessRow { id: "kev", @@ -190,6 +221,7 @@ pub(crate) fn derive_readiness( enable: "PROTECTOR_KEV_FILE", detail: coverage_detail(config.kev_count, "known-exploited CVE id"), weakens_decisions: true, + nodes: Vec::new(), }, ReadinessRow { id: "epss", @@ -199,24 +231,20 @@ pub(crate) fn derive_readiness( enable: "PROTECTOR_EPSS_FILE", detail: coverage_detail(config.epss_count, "EPSS score"), weakens_decisions: true, + nodes: Vec::new(), }, + // ONE agent-sourced runtime-corroboration row (JEF-308), replacing the former + // `falco` + `ebpf-agent` split. Its state IS the honesty ladder; its `nodes` carry the + // per-node breakdown the view renders as a `
`/`
`. ReadinessRow { - id: "falco", - label: "Falco feed", - state: falco_state, - why: "live rule-fired alerts confirm a path is being exploited right now", - enable: "runtime ingest (falcosidekick -> /alert)", - detail: signals_detail(falco_state, falco_signals), - weakens_decisions: true, - }, - ReadinessRow { - id: "ebpf-agent", - label: "eBPF agent", - state: ebpf_state, - why: "in-kernel behavioral signals (exec, secret reads, connections) show live activity", + id: "runtime-corroboration", + label: "Runtime corroboration", + state: runtime_state, + why: "live per-node behavioral signals (exec, secret reads, connections, alerts) confirm a path is being exploited right now — a blind node cannot corroborate", enable: "deploy the agent DaemonSet (-> /behavior)", - detail: signals_detail(ebpf_state, ebpf_signals), + detail: runtime_detail, weakens_decisions: true, + nodes: runtime_nodes, }, ReadinessRow { id: "journal", @@ -230,6 +258,7 @@ pub(crate) fn derive_readiness( "in-memory only — resets on restart".to_string() }, weakens_decisions: false, + nodes: Vec::new(), }, ReadinessRow { id: "tuf-root", @@ -241,6 +270,7 @@ pub(crate) fn derive_readiness( // Signing-detection trust, not a model-adjudication enrichment input — its absence // doesn't weaken the model's exploitability call (ADR-0016), so this stays false. weakens_decisions: false, + nodes: Vec::new(), }, ReadinessRow { id: "arm-state", @@ -256,6 +286,7 @@ pub(crate) fn derive_readiness( "shadow (proposing only)".to_string() }, weakens_decisions: false, + nodes: Vec::new(), }, ]; @@ -339,144 +370,169 @@ fn coverage_detail(count: usize, noun: &str) -> String { } } -/// The live detail for a behavioral feed: "N signals last pass", or an honest "none this -/// pass" when absent (no sensor reporting, or a quiet cluster). -fn signals_detail(state: InputState, signals: u64) -> String { - match state { - InputState::Present => format!( - "{signals} signal{} last pass", - if signals == 1 { "" } else { "s" } - ), - _ => "no signals last pass (no sensor reporting, or a quiet cluster)".to_string(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn covered_config() -> ReadinessConfig { - ReadinessConfig { - model_attached: true, - kev_count: 5, - epss_count: 5, - journal_durable: true, - armed: false, - // A fresh trust root, no fleet-wide unverifiable spike ⇒ the TUF row is Present. - tuf_cache_age_secs: Some(60), - unverifiable_spike: false, - } - } +/// Build the collapsed **Runtime corroboration** row (JEF-308) — its coarse [`InputState`], the +/// honest ladder detail, and the per-node breakdown — from the derived coverage and this pass's +/// Falco alert volume. The ladder, in order of increasing coverage: +/// +/// * **blind** (`Absent`) — no live signal at all: the agent is deployed nowhere in scope and +/// Falco is silent, OR every expected node is dark. Named honestly; absence of a signal is +/// never reassuring. +/// * **degraded** (`Degraded`) — SOME expected nodes are blind (named) or only partially +/// probing, while others are healthy (and/or Falco still feeds). +/// * **healthy** (`Present`) — every expected node is reporting with its probes loaded (quiet +/// counts). Falco still feeding is noted as the cutover "both-sources" rung. +/// +/// A node the agent isn't scheduled on is out-of-scope (not in the expected set), so it never +/// pushes the row off green. +fn runtime_corroboration_row( + runtime: &RuntimeCoverage, + falco_signals: u64, +) -> (InputState, String, Vec) { + let nodes = node_rows(runtime); + let falco_active = falco_signals > 0; + let expected = runtime.expected_count(); + let blind = runtime.blind_nodes(); + let degraded = runtime.degraded_nodes(); + let healthy = runtime.healthy_count(); + let agent_signals = runtime.agent_signals(); + + let falco_note = if falco_active { + format!( + " (+ Falco corroborating: {falco_signals} alert{} this pass, cutover)", + plural(falco_signals) + ) + } else { + String::new() + }; - #[test] - fn fully_covered_model_judging_has_no_unmet_inputs() { - let mut bake = BakeStats::default(); - bake.signals_by_variant.insert("alert".into(), 1); - bake.signals_by_variant.insert("connection".into(), 1); - let readiness = derive_readiness( - &covered_config(), - ModelHealth::Ok, - &bake, - Some(SystemTime::now()), - ); - assert!(readiness.model_judging); - assert!(readiness.model_attached()); - assert!(!readiness.has_unmet()); - assert_eq!(readiness.unmet_count(), 0); - assert!(!readiness.warming_up); + // No expected nodes: the agent isn't scheduled anywhere in scope this pass. + if expected == 0 { + return if falco_active { + ( + InputState::Present, + format!( + "Falco corroborating ({falco_signals} alert{} this pass) — eBPF agent not reporting on any node (cutover)", + plural(falco_signals) + ), + nodes, + ) + } else { + ( + InputState::Absent, + "BLIND: no runtime sensor reporting — absence of a signal is not evidence of safety" + .to_string(), + nodes, + ) + }; } - #[test] - fn a_timed_out_model_is_degraded_not_judging() { - let readiness = derive_readiness( - &covered_config(), - ModelHealth::Timeout, - &BakeStats::default(), - Some(SystemTime::now()), + // Every expected node is dark AND Falco is silent → wholly blind. + if blind.len() == expected && !falco_active { + return ( + InputState::Absent, + format!( + "BLIND: all {expected} expected node{} dark ({}) — corroboration has no live sensor", + plural(expected as u64), + blind.join(", ") + ), + nodes, ); - assert!(!readiness.model_judging); - // The model is still CONFIGURED — attached, just not answering. - assert!(readiness.model_attached()); - // The model row is degraded and the (quiet) behavioral feeds are absent ⇒ unmet. - assert!(readiness.has_unmet()); } - /// The TUF-root row from a readiness snapshot. - fn tuf(readiness: &Readiness) -> &ReadinessRow { - readiness - .inputs - .iter() - .find(|r| r.id == "tuf-root") - .expect("a TUF-root row is present") - } - - #[test] - fn a_fresh_trust_root_reads_present() { - let readiness = derive_readiness( - &covered_config(), - ModelHealth::Ok, - &BakeStats::default(), - Some(SystemTime::now()), + // Some expected nodes blind → degraded, naming them. + if !blind.is_empty() { + return ( + InputState::Degraded, + format!( + "degraded — blind on {} of {expected} node{}: {} ({healthy} healthy){falco_note}", + blind.len(), + plural(expected as u64), + blind.join(", ") + ), + nodes, ); - assert_eq!(tuf(&readiness).state, InputState::Present); - assert!(tuf(&readiness).detail.contains("fresh")); } - #[test] - fn a_stale_trust_root_is_degraded_and_surfaced_non_green() { - let mut config = covered_config(); - config.tuf_cache_age_secs = Some(TUF_STALE_AFTER_SECS + 1); - let readiness = derive_readiness( - &config, - ModelHealth::Ok, - &BakeStats::default(), - Some(SystemTime::now()), + // Only partial-probe degradation remains. + if !degraded.is_empty() { + return ( + InputState::Degraded, + format!( + "degraded — probes partial on {}: {}{falco_note}", + degraded.len(), + degraded.join(", ") + ), + nodes, ); - assert_eq!(tuf(&readiness).state, InputState::Degraded); - assert!(tuf(&readiness).detail.contains("stale")); - // Non-green: a stale trust root counts as an unmet input. - assert!(readiness.has_unmet()); } - #[test] - fn a_never_fetched_trust_root_reads_absent() { - let mut config = covered_config(); - config.tuf_cache_age_secs = None; - let readiness = derive_readiness( - &config, - ModelHealth::Ok, - &BakeStats::default(), - Some(SystemTime::now()), - ); - assert_eq!(tuf(&readiness).state, InputState::Absent); - } + // Every expected node healthy (quiet counts). + ( + InputState::Present, + format!( + "healthy — all {expected} node{} reporting, probes loaded ({agent_signals} signal{} this pass){falco_note}", + plural(expected as u64), + plural(agent_signals) + ), + nodes, + ) +} - #[test] - fn a_fleet_wide_unverifiable_spike_is_surfaced_even_on_a_fresh_root() { - let mut config = covered_config(); - config.tuf_cache_age_secs = Some(60); // fresh mtime … - config.unverifiable_spike = true; // … but a mass unverifiable spike this pass - let readiness = derive_readiness( - &config, - ModelHealth::Ok, - &BakeStats::default(), - Some(SystemTime::now()), - ); - assert_eq!(tuf(&readiness).state, InputState::Degraded); - assert!(tuf(&readiness).detail.contains("spike")); - assert!(readiness.has_unmet()); - } +/// Project the derived coverage into the per-node display rows (JEF-308). Each line names the +/// node, its honest state, and a short live detail — quiet is spelled out as quiet, blind names +/// its reason, so the two never collapse into one word. +fn node_rows(runtime: &RuntimeCoverage) -> Vec { + runtime + .nodes + .iter() + .map(|n| { + let (state, detail) = match n.state { + NodeState::Healthy { signals: 0 } => ( + NodeCoverageState::Healthy, + "quiet — probes loaded, 0 signals this pass".to_string(), + ), + NodeState::Healthy { signals } => ( + NodeCoverageState::Healthy, + format!( + "{signals} signal{} this pass, probes loaded", + plural(signals) + ), + ), + NodeState::DegradedProbes { loaded, total } => ( + NodeCoverageState::Degraded, + format!("partial — {loaded}/{total} probes loaded"), + ), + NodeState::Blind { + reason: BlindReason::NotReporting, + } => ( + NodeCoverageState::Blind, + "not reporting — no live sensor on this node".to_string(), + ), + NodeState::Blind { + reason: BlindReason::ProbesFailed, + } => ( + NodeCoverageState::Blind, + "probes failed to load — agent up but blind".to_string(), + ), + NodeState::OutOfScope => ( + NodeCoverageState::OutOfScope, + "agent not scheduled here".to_string(), + ), + }; + NodeCoverageRow { + node: n.node.clone(), + state, + detail, + } + }) + .collect() +} - #[test] - fn an_unconfigured_model_reads_absent_and_warming_before_first_pass() { - let readiness = derive_readiness( - &ReadinessConfig::default(), - ModelHealth::Unknown, - &BakeStats::default(), - None, - ); - assert!(!readiness.model_attached()); - assert!(!readiness.model_judging); - assert!(readiness.warming_up); - } +/// `"s"` unless `n == 1` — the pluralization for the honest count lines. +fn plural(n: u64) -> &'static str { + if n == 1 { "" } else { "s" } } + +#[cfg(test)] +#[path = "readiness_tests.rs"] +mod tests; diff --git a/engine/src/engine/state/readiness_tests.rs b/engine/src/engine/state/readiness_tests.rs new file mode 100644 index 0000000..20b6def --- /dev/null +++ b/engine/src/engine/state/readiness_tests.rs @@ -0,0 +1,343 @@ +//! Tests for the readiness aggregation (JEF-160) and the collapsed runtime-corroboration row +//! (JEF-308). Extracted to keep `readiness.rs` under the 1,000-line cap (CLAUDE.md). + +use super::super::agent_liveness::{BlindReason, NodeCoverage, NodeState, RuntimeCoverage}; +use super::*; + +fn covered_config() -> ReadinessConfig { + ReadinessConfig { + model_attached: true, + kev_count: 5, + epss_count: 5, + journal_durable: true, + armed: false, + // A fresh trust root, no fleet-wide unverifiable spike ⇒ the TUF row is Present. + tuf_cache_age_secs: Some(60), + unverifiable_spike: false, + } +} + +/// An empty runtime-coverage snapshot (no expected agent nodes) — the default for tests that +/// aren't exercising the per-node liveness path. +fn no_runtime() -> RuntimeCoverage { + RuntimeCoverage::default() +} + +/// A coverage snapshot from `(node, state)` pairs. +fn coverage(nodes: &[(&str, NodeState)]) -> RuntimeCoverage { + RuntimeCoverage { + nodes: nodes + .iter() + .map(|(n, s)| NodeCoverage { + node: (*n).to_string(), + state: *s, + }) + .collect(), + } +} + +/// The runtime-corroboration row from a readiness snapshot. +fn runtime_row(readiness: &Readiness) -> &ReadinessRow { + readiness + .inputs + .iter() + .find(|r| r.id == "runtime-corroboration") + .expect("a runtime-corroboration row is present") +} + +#[test] +fn fully_covered_model_judging_has_no_unmet_inputs() { + let mut bake = BakeStats::default(); + bake.signals_by_variant.insert("alert".into(), 1); + bake.signals_by_variant.insert("connection".into(), 1); + // One healthy expected node → runtime corroboration is Present. + let cov = coverage(&[("node-a", NodeState::Healthy { signals: 2 })]); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &cov, + ); + assert!(readiness.model_judging); + assert!(readiness.model_attached()); + assert!(!readiness.has_unmet()); + assert_eq!(readiness.unmet_count(), 0); + assert!(!readiness.warming_up); +} + +#[test] +fn a_timed_out_model_is_degraded_not_judging() { + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Timeout, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + assert!(!readiness.model_judging); + // The model is still CONFIGURED — attached, just not answering. + assert!(readiness.model_attached()); + // The model row is degraded and the (quiet) behavioral feeds are absent ⇒ unmet. + assert!(readiness.has_unmet()); +} + +/// The TUF-root row from a readiness snapshot. +fn tuf(readiness: &Readiness) -> &ReadinessRow { + readiness + .inputs + .iter() + .find(|r| r.id == "tuf-root") + .expect("a TUF-root row is present") +} + +#[test] +fn a_fresh_trust_root_reads_present() { + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + assert_eq!(tuf(&readiness).state, InputState::Present); + assert!(tuf(&readiness).detail.contains("fresh")); +} + +#[test] +fn a_stale_trust_root_is_degraded_and_surfaced_non_green() { + let mut config = covered_config(); + config.tuf_cache_age_secs = Some(TUF_STALE_AFTER_SECS + 1); + let readiness = derive_readiness( + &config, + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + assert_eq!(tuf(&readiness).state, InputState::Degraded); + assert!(tuf(&readiness).detail.contains("stale")); + // Non-green: a stale trust root counts as an unmet input. + assert!(readiness.has_unmet()); +} + +#[test] +fn a_never_fetched_trust_root_reads_absent() { + let mut config = covered_config(); + config.tuf_cache_age_secs = None; + let readiness = derive_readiness( + &config, + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + assert_eq!(tuf(&readiness).state, InputState::Absent); +} + +#[test] +fn a_fleet_wide_unverifiable_spike_is_surfaced_even_on_a_fresh_root() { + let mut config = covered_config(); + config.tuf_cache_age_secs = Some(60); // fresh mtime … + config.unverifiable_spike = true; // … but a mass unverifiable spike this pass + let readiness = derive_readiness( + &config, + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + assert_eq!(tuf(&readiness).state, InputState::Degraded); + assert!(tuf(&readiness).detail.contains("spike")); + assert!(readiness.has_unmet()); +} + +#[test] +fn an_unconfigured_model_reads_absent_and_warming_before_first_pass() { + let readiness = derive_readiness( + &ReadinessConfig::default(), + ModelHealth::Unknown, + &BakeStats::default(), + None, + &no_runtime(), + ); + assert!(!readiness.model_attached()); + assert!(!readiness.model_judging); + assert!(readiness.warming_up); +} + +// --- JEF-308: the collapsed runtime-corroboration row + honesty ladder --- + +/// There is exactly ONE runtime row now — the former `falco` + `ebpf-agent` split is collapsed. +#[test] +fn runtime_corroboration_is_a_single_agent_sourced_row() { + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + assert!(readiness.inputs.iter().all(|r| r.id != "falco")); + assert!(readiness.inputs.iter().all(|r| r.id != "ebpf-agent")); + assert_eq!( + readiness + .inputs + .iter() + .filter(|r| r.id == "runtime-corroboration") + .count(), + 1 + ); +} + +#[test] +fn all_nodes_healthy_reads_present() { + let cov = coverage(&[ + ("node-a", NodeState::Healthy { signals: 3 }), + ("node-b", NodeState::Healthy { signals: 0 }), // quiet — still healthy + ]); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &cov, + ); + let row = runtime_row(&readiness); + assert_eq!(row.state, InputState::Present); + assert!(row.detail.starts_with("healthy")); + assert_eq!(row.nodes.len(), 2); + // The quiet node is spelled out as quiet, not absent. + let quiet = row.nodes.iter().find(|n| n.node == "node-b").unwrap(); + assert_eq!(quiet.state, NodeCoverageState::Healthy); + assert!(quiet.detail.contains("quiet")); +} + +#[test] +fn a_blind_node_degrades_the_row_and_is_named() { + let cov = coverage(&[ + ("node-a", NodeState::Healthy { signals: 1 }), + ( + "node-b", + NodeState::Blind { + reason: BlindReason::NotReporting, + }, + ), + ]); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &cov, + ); + let row = runtime_row(&readiness); + assert_eq!(row.state, InputState::Degraded); + assert!(row.detail.contains("node-b"), "the blind node is named"); + assert!(readiness.has_unmet()); + let b = row.nodes.iter().find(|n| n.node == "node-b").unwrap(); + assert_eq!(b.state, NodeCoverageState::Blind); +} + +#[test] +fn probes_failed_node_reads_blind_despite_reporting() { + let cov = coverage(&[( + "node-a", + NodeState::Blind { + reason: BlindReason::ProbesFailed, + }, + )]); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &cov, + ); + let row = runtime_row(&readiness); + // The single expected node is dark and Falco is silent ⇒ wholly BLIND (Absent). + assert_eq!(row.state, InputState::Absent); + assert!(row.detail.contains("BLIND")); + let a = row.nodes.iter().find(|n| n.node == "node-a").unwrap(); + assert!(a.detail.contains("probes failed")); +} + +#[test] +fn no_sensor_at_all_reads_blind_never_reassuring() { + // No expected agent nodes and Falco silent → the row is BLIND, and its detail says absence of + // a signal is not evidence of safety (the honesty invariant). + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &no_runtime(), + ); + let row = runtime_row(&readiness); + assert_eq!(row.state, InputState::Absent); + assert!(row.detail.contains("not evidence of safety")); + assert!(readiness.has_unmet()); +} + +#[test] +fn falco_only_during_cutover_reads_present_with_a_cutover_note() { + // Falco still feeding but the agent isn't reporting on any node yet — the both-sources rung. + let mut bake = BakeStats::default(); + bake.signals_by_variant.insert("alert".into(), 2); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &bake, + Some(SystemTime::now()), + &no_runtime(), + ); + let row = runtime_row(&readiness); + assert_eq!(row.state, InputState::Present); + assert!(row.detail.contains("Falco")); + assert!(row.detail.contains("cutover")); +} + +#[test] +fn a_partial_probe_node_is_degraded_not_blind() { + let cov = coverage(&[( + "node-a", + NodeState::DegradedProbes { + loaded: 4, + total: 6, + }, + )]); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &cov, + ); + let row = runtime_row(&readiness); + assert_eq!(row.state, InputState::Degraded); + assert!(row.detail.contains("partial")); + let a = &row.nodes[0]; + assert_eq!(a.state, NodeCoverageState::Degraded); + assert!(a.detail.contains("4/6")); +} + +#[test] +fn an_out_of_scope_reporter_does_not_push_the_row_off_green() { + // An out-of-scope reporter (agent seen where it isn't scheduled) is not blind and doesn't + // count as an expected node — one healthy expected node keeps the row Present. + let cov = coverage(&[ + ("node-a", NodeState::Healthy { signals: 1 }), + ("node-x", NodeState::OutOfScope), + ]); + let readiness = derive_readiness( + &covered_config(), + ModelHealth::Ok, + &BakeStats::default(), + Some(SystemTime::now()), + &cov, + ); + let row = runtime_row(&readiness); + assert_eq!(row.state, InputState::Present); + let x = row.nodes.iter().find(|n| n.node == "node-x").unwrap(); + assert_eq!(x.state, NodeCoverageState::OutOfScope); +} diff --git a/engine/src/engine/tests.rs b/engine/src/engine/tests.rs index 54e5af5..c7c34a9 100644 --- a/engine/src/engine/tests.rs +++ b/engine/src/engine/tests.rs @@ -319,6 +319,7 @@ async fn corroboration_predicate_fires_on_a_live_alert() { attribution: Attribution::by_namespaced_name("app", "web"), source: Some("falco".into()), observed_at_ms: None, + node: None, behavior: Behavior::Alert { rule: "Terminal shell in container".into(), }, @@ -352,6 +353,7 @@ async fn process_publishes_the_behavioral_bake_snapshot() { attribution: Attribution::by_namespaced_name("app", "web"), source: Some("falco".into()), observed_at_ms: None, + node: None, behavior: Behavior::Alert { rule: "Terminal shell in container".into(), }, @@ -361,6 +363,7 @@ async fn process_publishes_the_behavioral_bake_snapshot() { attribution: Attribution::by_pod_uid("uid-not-in-snapshot"), source: Some("agent".into()), observed_at_ms: None, + node: None, behavior: Behavior::NetworkConnection { peer: "10.0.0.9:443".into(), internet: false,