From 9c9222953aa713014e15c5f7f657d56402e146c4 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:02:52 +0200 Subject: [PATCH 01/30] =?UTF-8?q?feat(stage-a-io):=20=E2=9C=A8=20add=20sha?= =?UTF-8?q?red=20Stage-A=20Teensy=20I/O=20library?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PDA1 wire protocol (fragmentation-tolerant parser with CRC resync), v1 ASCII command grammar, typed serial client with idempotent sequence retries and stream-integrity accounting, bounded background I/O worker, .pdq raw-frame writer, JSON run sidecar, calibrated optical log-contrast estimator (clipping/dark-headroom guarded), and a mock controller for hardware-free tests. Wire-compatible with stage-a-controller include/wire_protocol.h. --- Cargo.toml | 1 + stage-a-io/Cargo.toml | 18 ++ stage-a-io/src/client.rs | 329 +++++++++++++++++++++++++++ stage-a-io/src/estimator.rs | 238 ++++++++++++++++++++ stage-a-io/src/lib.rs | 43 ++++ stage-a-io/src/mock.rs | 283 +++++++++++++++++++++++ stage-a-io/src/pdq.rs | 150 ++++++++++++ stage-a-io/src/protocol.rs | 226 +++++++++++++++++++ stage-a-io/src/sidecar.rs | 216 ++++++++++++++++++ stage-a-io/src/transport.rs | 113 ++++++++++ stage-a-io/src/wire.rs | 438 ++++++++++++++++++++++++++++++++++++ stage-a-io/src/worker.rs | 229 +++++++++++++++++++ 12 files changed, 2284 insertions(+) create mode 100644 stage-a-io/Cargo.toml create mode 100644 stage-a-io/src/client.rs create mode 100644 stage-a-io/src/estimator.rs create mode 100644 stage-a-io/src/lib.rs create mode 100644 stage-a-io/src/mock.rs create mode 100644 stage-a-io/src/pdq.rs create mode 100644 stage-a-io/src/protocol.rs create mode 100644 stage-a-io/src/sidecar.rs create mode 100644 stage-a-io/src/transport.rs create mode 100644 stage-a-io/src/wire.rs create mode 100644 stage-a-io/src/worker.rs diff --git a/Cargo.toml b/Cargo.toml index 306f834..98ff631 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "stage-a-io", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/stage-a-io/Cargo.toml b/stage-a-io/Cargo.toml new file mode 100644 index 0000000..7a8d9fa --- /dev/null +++ b/stage-a-io/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "stage-a-io" +description = "Shared Stage-A Teensy I/O: PDA1 wire protocol, serial client, PDQ writer, run sidecars, and the calibrated optical-contrast estimator" +edition.workspace = true +license.workspace = true +version.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +serialport = { version = "4", optional = true } + +[features] +default = ["hardware"] +# Real serial-port transport. Disable for pure-analysis / CI builds. +hardware = ["dep:serialport"] diff --git a/stage-a-io/src/client.rs b/stage-a-io/src/client.rs new file mode 100644 index 0000000..c664be0 --- /dev/null +++ b/stage-a-io/src/client.rs @@ -0,0 +1,329 @@ +//! Typed request/response client over a [`Transport`]. +//! +//! Sends `@ VERB …` commands and demultiplexes the PDA1 frame stream +//! into (a) the matching control reply, (b) async control notices, and +//! (c) data frames (samples / summaries / markers). On a reply timeout the +//! **identical** line (same `seq`) is resent; firmware caches recent replies, +//! so retries are idempotent by construction. + +use std::collections::BTreeMap; +use std::io; +use std::time::{Duration, Instant}; + +use crate::protocol::{Command, ControlMessage, ProtocolError}; +use crate::transport::Transport; +use crate::wire::{Frame, FrameParser, FrameType, ParseEvent}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct StreamIntegrity { + pub skipped_bytes: u64, + pub crc_failures: u64, + pub sequence_gaps: u64, + pub dropped_samples: u64, +} + +impl StreamIntegrity { + /// A run is valid only while the stream shows zero corruption. + pub fn is_clean(&self) -> bool { + self.skipped_bytes == 0 + && self.crc_failures == 0 + && self.sequence_gaps == 0 + && self.dropped_samples == 0 + } +} + +#[derive(Debug)] +pub enum ClientError { + Io(io::Error), + Protocol(ProtocolError), + /// The device replied `-seq ERR …`. + Device { + code: String, + detail: String, + }, + /// No matching reply within the timeout across all retries. + Timeout { + verb: String, + retries: u32, + }, +} + +impl std::fmt::Display for ClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(err) => write!(f, "transport I/O failed: {err}"), + Self::Protocol(err) => write!(f, "protocol violation: {err}"), + Self::Device { code, detail } => { + write!(f, "device rejected command: code={code} detail={detail}") + } + Self::Timeout { verb, retries } => { + write!(f, "no reply to {verb} after {retries} retries") + } + } + } +} + +impl std::error::Error for ClientError {} + +impl From for ClientError { + fn from(err: io::Error) -> Self { + Self::Io(err) + } +} + +impl From for ClientError { + fn from(err: ProtocolError) -> Self { + Self::Protocol(err) + } +} + +/// Non-reply traffic observed while waiting for or between replies. +#[derive(Debug, Clone, PartialEq)] +pub enum DeviceEvent { + Data(Frame), + Async { + name: String, + fields: BTreeMap, + }, +} + +pub struct StageAClient { + transport: T, + parser: FrameParser, + next_sequence: u32, + last_frame_sequence: Option, + integrity: StreamIntegrity, + pending_events: Vec, + reply_timeout: Duration, + max_retries: u32, + read_buf: Vec, +} + +impl StageAClient { + pub fn new(transport: T) -> Self { + Self { + transport, + parser: FrameParser::default(), + next_sequence: 1, + last_frame_sequence: None, + integrity: StreamIntegrity::default(), + pending_events: Vec::new(), + reply_timeout: Duration::from_millis(500), + max_retries: 2, + read_buf: vec![0_u8; 16 * 1024], + } + } + + pub fn with_reply_timeout(mut self, timeout: Duration) -> Self { + self.reply_timeout = timeout; + self + } + + pub fn integrity(&self) -> StreamIntegrity { + self.integrity + } + + /// Sends a command and waits for its `+seq OK` reply, retrying the + /// identical line on timeout. Data/async frames arriving in between are + /// queued for [`StageAClient::poll_events`]. + pub fn request(&mut self, command: &Command) -> Result, ClientError> { + let sequence = self.next_sequence; + self.next_sequence = self.next_sequence.wrapping_add(1); + let line = command.encode(sequence)?; + + for _attempt in 0..=self.max_retries { + self.transport.write_all(&line)?; + let deadline = Instant::now() + self.reply_timeout; + while Instant::now() < deadline { + self.pump()?; + if let Some(reply) = self.take_reply(sequence)? { + return Ok(reply); + } + std::thread::sleep(Duration::from_millis(1)); + } + } + Err(ClientError::Timeout { + verb: command.verb.clone(), + retries: self.max_retries, + }) + } + + /// Drains any pending non-reply device traffic (data frames, async + /// notices) without blocking. + pub fn poll_events(&mut self) -> Result, ClientError> { + self.pump()?; + Ok(std::mem::take(&mut self.pending_events)) + } + + fn pump(&mut self) -> Result<(), ClientError> { + let n = self.transport.read(&mut self.read_buf)?; + if n > 0 { + self.parser.extend(&self.read_buf[..n]); + } + while let Some(event) = self.parser.next_event() { + match event { + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + self.integrity.skipped_bytes += skipped_bytes as u64; + self.integrity.crc_failures += crc_failures as u64; + } + ParseEvent::Frame(frame) => self.accept_frame(frame), + } + } + Ok(()) + } + + fn accept_frame(&mut self, frame: Frame) { + if let Some(last) = self.last_frame_sequence { + let expected = last.wrapping_add(1); + if frame.header.sequence != expected { + self.integrity.sequence_gaps += 1; + } + } + self.last_frame_sequence = Some(frame.header.sequence); + if frame.header.dropped_samples > 0 { + self.integrity.dropped_samples = u64::from(frame.header.dropped_samples); + } + + match frame.header.frame_type { + FrameType::Control => { + // Control payloads are handled by take_reply / async queue; + // keep the raw frame so replies can be matched later. + self.pending_events.push(DeviceEvent::Data(frame)); + } + _ => self.pending_events.push(DeviceEvent::Data(frame)), + } + } + + fn take_reply( + &mut self, + sequence: u32, + ) -> Result>, ClientError> { + let mut result = None; + let mut remaining = Vec::with_capacity(self.pending_events.len()); + for event in std::mem::take(&mut self.pending_events) { + if result.is_some() { + remaining.push(event); + continue; + } + let DeviceEvent::Data(frame) = &event else { + remaining.push(event); + continue; + }; + let Some(text) = frame.control_text() else { + remaining.push(event); + continue; + }; + match ControlMessage::parse(text) { + Ok(ControlMessage::Ok { + sequence: reply_seq, + fields, + }) if reply_seq == sequence => { + result = Some(Ok(fields)); + } + Ok(ControlMessage::Err { + sequence: reply_seq, + code, + detail, + }) if reply_seq == sequence => { + result = Some(Err(ClientError::Device { code, detail })); + } + Ok(ControlMessage::Async { name, fields }) => { + remaining.push(DeviceEvent::Async { name, fields }); + } + // Stale replies to earlier (retried) sequences are dropped; + // malformed control payloads count as corruption. + Ok(_) => {} + Err(_) => { + self.integrity.crc_failures += 0; // parse failure, not CRC + self.integrity.skipped_bytes += frame.payload.len() as u64; + } + } + } + self.pending_events = remaining; + match result { + Some(Ok(fields)) => Ok(Some(fields)), + Some(Err(err)) => Err(err), + None => Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::MockController; + use crate::transport::MockLink; + + #[test] + fn request_reply_round_trip_with_hello() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + let handle = std::thread::spawn(move || controller.serve_n_commands(1)); + let reply = client + .request(&Command::new("HELLO").field("protocol", 1)) + .expect("HELLO replies"); + handle.join().expect("mock thread joins"); + + assert_eq!(reply.get("protocol").map(String::as_str), Some("1")); + assert!(client.integrity().is_clean()); + } + + #[test] + fn timeout_retries_are_idempotent_via_reply_cache() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + controller.drop_first_reply(); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(50)); + + // The controller swallows the first reply; the client must resend the + // identical sequence and accept the cached second reply. The mock + // panics if a retried sequence re-executes the operation. + let handle = std::thread::spawn(move || controller.serve_n_commands(2)); + let reply = client + .request(&Command::new("STATUS")) + .expect("retried STATUS succeeds"); + handle.join().expect("mock thread joins"); + + assert_eq!(reply.get("state").map(String::as_str), Some("SAFE_IDLE")); + assert_eq!(reply.get("executions").map(String::as_str), Some("1")); + } + + #[test] + fn device_error_reply_surfaces_code_and_detail() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + let handle = std::thread::spawn(move || controller.serve_n_commands(1)); + let err = client + .request(&Command::new("CONFIG").field("mode", "A9")) + .expect_err("invalid mode is rejected"); + handle.join().expect("mock thread joins"); + + match err { + ClientError::Device { code, .. } => assert_eq!(code, "BAD_MODE"), + other => panic!("expected device error, got {other:?}"), + } + } + + #[test] + fn overrun_frames_invalidate_integrity() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + controller.emit_summary_with_drops(3); + client.poll_events().expect("poll"); + assert!(!client.integrity().is_clean()); + assert_eq!(client.integrity().dropped_samples, 3); + } +} diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs new file mode 100644 index 0000000..809786a --- /dev/null +++ b/stage-a-io/src/estimator.rs @@ -0,0 +1,238 @@ +//! Calibrated optical log-contrast estimator. +//! +//! `a = ln(I_max / I_min)` is defined by the *measured light*, never by the +//! commanded DAC excursion: the Pockels-cell V→T response is non-linear, so +//! the photodiode ADC trace is the only valid source of `a` +//! (knowledge base: `methodology/camera-calibration.md`, "define `a` from +//! the light, not the drive"). +//! +//! The estimator therefore: +//! - converts ADC codes to volts through a characterised affine calibration, +//! - subtracts the dark level (the detector is DC-coupled; `a` needs true +//! levels including DC), +//! - takes robust percentile extrema rather than raw min/max so single-code +//! noise spikes do not bias the contrast, +//! - refuses to produce a value at all when the window clips (top/bottom of +//! the ADC range) or has no headroom above dark — a wrong `a` is worse +//! than no `a`. + +use serde::{Deserialize, Serialize}; + +/// Affine ADC calibration plus dark level, all in physical units. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AdcCalibration { + /// Volts per ADC code (gain of the whole front end into the ADC). + pub volts_per_code: f64, + /// Voltage at code 0. + pub offset_volts: f64, + /// Dark level (light blocked), in volts after the affine map. + pub dark_volts: f64, + /// Full-scale code (4095 for the Teensy 12-bit ADC). + pub full_scale_code: u16, +} + +impl Default for AdcCalibration { + fn default() -> Self { + Self { + volts_per_code: 3.3 / 4_095.0, + offset_volts: 0.0, + dark_volts: 0.0, + full_scale_code: 4_095, + } + } +} + +impl AdcCalibration { + pub fn code_to_volts(&self, code: u16) -> f64 { + self.offset_volts + f64::from(code) * self.volts_per_code + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ContrastEstimate { + /// Peak-to-peak log-contrast `a = ln(V_max / V_min)` (dark-corrected). + pub a: f64, + pub v_min_volts: f64, + pub v_max_volts: f64, + /// Fraction of samples at or below code 0 + margin. + pub low_clip_fraction: f64, + /// Fraction of samples at or above full scale - margin. + pub high_clip_fraction: f64, + pub sample_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EstimateError { + /// Fewer samples than the estimator can use robustly. + TooFewSamples { count: usize, minimum: usize }, + /// The window touches the ADC rails — `a` would be silently wrong. + Clipped { + low_fraction_permille: u32, + high_fraction_permille: u32, + }, + /// The dark-corrected minimum is not positive: no optical headroom. + NoHeadroomAboveDark, +} + +impl std::fmt::Display for EstimateError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooFewSamples { count, minimum } => { + write!(f, "only {count} samples (minimum {minimum})") + } + Self::Clipped { + low_fraction_permille, + high_fraction_permille, + } => write!( + f, + "ADC clipping: {low_fraction_permille}‰ low / {high_fraction_permille}‰ high" + ), + Self::NoHeadroomAboveDark => { + f.write_str("dark-corrected minimum is not positive; a is undefined") + } + } + } +} + +impl std::error::Error for EstimateError {} + +pub const MIN_SAMPLES: usize = 64; +/// Codes within this margin of the rails count as clipped. +pub const CLIP_MARGIN_CODES: u16 = 4; +/// Reject the window when more than 1‰ of samples clip. +pub const MAX_CLIP_FRACTION: f64 = 0.001; +/// Robust extrema: 1st / 99th percentile. +const LOW_PERCENTILE: f64 = 0.01; +const HIGH_PERCENTILE: f64 = 0.99; + +/// Estimates the optical log-contrast from one settled, phase-attributed +/// ADC window. The window must span at least a few full modulation cycles; +/// enforcing that is the caller's job (it knows the drive frequency). +pub fn estimate_contrast( + codes: &[u16], + calibration: &AdcCalibration, +) -> Result { + if codes.len() < MIN_SAMPLES { + return Err(EstimateError::TooFewSamples { + count: codes.len(), + minimum: MIN_SAMPLES, + }); + } + + let low_clip_threshold = CLIP_MARGIN_CODES; + let high_clip_threshold = calibration + .full_scale_code + .saturating_sub(CLIP_MARGIN_CODES); + let low_clipped = codes.iter().filter(|&&c| c <= low_clip_threshold).count(); + let high_clipped = codes.iter().filter(|&&c| c >= high_clip_threshold).count(); + let low_clip_fraction = low_clipped as f64 / codes.len() as f64; + let high_clip_fraction = high_clipped as f64 / codes.len() as f64; + if low_clip_fraction > MAX_CLIP_FRACTION || high_clip_fraction > MAX_CLIP_FRACTION { + return Err(EstimateError::Clipped { + low_fraction_permille: (low_clip_fraction * 1_000.0).round() as u32, + high_fraction_permille: (high_clip_fraction * 1_000.0).round() as u32, + }); + } + + let mut sorted = codes.to_vec(); + sorted.sort_unstable(); + let low_code = percentile(&sorted, LOW_PERCENTILE); + let high_code = percentile(&sorted, HIGH_PERCENTILE); + + let v_min = calibration.code_to_volts(low_code) - calibration.dark_volts; + let v_max = calibration.code_to_volts(high_code) - calibration.dark_volts; + if v_min <= 0.0 || v_max <= 0.0 { + return Err(EstimateError::NoHeadroomAboveDark); + } + + Ok(ContrastEstimate { + a: (v_max / v_min).ln(), + v_min_volts: v_min, + v_max_volts: v_max, + low_clip_fraction, + high_clip_fraction, + sample_count: codes.len(), + }) +} + +fn percentile(sorted: &[u16], q: f64) -> u16 { + let index = ((sorted.len() - 1) as f64 * q).round() as usize; + sorted[index.min(sorted.len() - 1)] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sine_codes(center: f64, amplitude: f64, n: usize) -> Vec { + (0..n) + .map(|i| { + let phase = 2.0 * std::f64::consts::PI * (i as f64) * 7.0 / n as f64; + (center + amplitude * phase.sin()) + .round() + .clamp(0.0, 4_095.0) as u16 + }) + .collect() + } + + #[test] + fn recovers_known_contrast_from_synthetic_sine() { + let calibration = AdcCalibration { + dark_volts: 40.0 * (3.3 / 4_095.0), + ..AdcCalibration::default() + }; + // center 2048, amplitude 900 -> dark-corrected V ratio: + let codes = sine_codes(2_048.0, 900.0, 4_096); + let estimate = estimate_contrast(&codes, &calibration).expect("clean window estimates"); + + let expected = ((2_048.0_f64 + 900.0 - 40.0) / (2_048.0 - 900.0 - 40.0)).ln(); + assert!( + (estimate.a - expected).abs() < 0.01, + "a={} expected~{expected}", + estimate.a + ); + assert!(estimate.low_clip_fraction == 0.0 && estimate.high_clip_fraction == 0.0); + } + + #[test] + fn rejects_clipped_windows() { + // Amplitude pushes past full scale -> clipping at the top rail. + let codes = sine_codes(3_500.0, 900.0, 2_048); + let err = estimate_contrast(&codes, &AdcCalibration::default()) + .expect_err("clipped window must be rejected"); + assert!(matches!(err, EstimateError::Clipped { .. })); + } + + #[test] + fn rejects_windows_without_dark_headroom() { + let calibration = AdcCalibration { + dark_volts: 1_300.0 * (3.3 / 4_095.0), + ..AdcCalibration::default() + }; + // Minimum (2048-900=1148) sits below the dark level (1300). + let codes = sine_codes(2_048.0, 900.0, 2_048); + let err = estimate_contrast(&codes, &calibration) + .expect_err("no headroom above dark must be rejected"); + assert_eq!(err, EstimateError::NoHeadroomAboveDark); + } + + #[test] + fn rejects_short_windows() { + let err = estimate_contrast(&[100; 10], &AdcCalibration::default()) + .expect_err("short window rejected"); + assert!(matches!(err, EstimateError::TooFewSamples { .. })); + } + + #[test] + fn single_sample_spikes_do_not_bias_the_contrast() { + let mut codes = sine_codes(2_048.0, 500.0, 4_096); + codes[7] = 4_000; // one hot spike, below the 1 - 99 percentile weight + let clean = estimate_contrast( + &sine_codes(2_048.0, 500.0, 4_096), + &AdcCalibration::default(), + ) + .expect("clean"); + let spiked = estimate_contrast(&codes, &AdcCalibration::default()).expect("spiked"); + assert!((clean.a - spiked.a).abs() < 0.005); + } +} diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs new file mode 100644 index 0000000..67c517f --- /dev/null +++ b/stage-a-io/src/lib.rs @@ -0,0 +1,43 @@ +//! # stage-a-io +//! +//! Shared research-owned I/O library for the Stage-A camera-calibration +//! plugins (`stage-a-monitor`, `stage-a-a1`, `stage-a-a2`, `stage-a-a3`). +//! +//! Scope, per the Stage-A control-software specification: +//! - the v1 ASCII command grammar and PDA1 binary frame format (wire- +//! compatible with `stage-a-controller/include/wire_protocol.h`), +//! - a typed serial client with idempotent sequence retries and stream- +//! integrity accounting (CRC failures, resync skips, sequence gaps, +//! ADC overruns — any of which invalidates a measurement point), +//! - a bounded background I/O worker so plugin `process_frame()` never +//! blocks on serial, +//! - the `.pdq` raw-frame writer and the JSON run sidecar, +//! - the calibrated optical log-contrast estimator (`a` is measured light, +//! never the commanded DAC excursion), +//! - a mock controller for tests and hardware-free development. +//! +//! This crate deliberately contains **no** experiment policy (sweeps, +//! bisection, fits live in the protocol plugins) and **no** augur types — +//! it is plain I/O + numerics, testable without a host. + +pub mod client; +pub mod estimator; +pub mod mock; +pub mod pdq; +pub mod protocol; +pub mod sidecar; +pub mod transport; +pub mod wire; + +pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; +pub use estimator::{estimate_contrast, AdcCalibration, ContrastEstimate, EstimateError}; +pub use pdq::{PdqSummary, PdqWriter}; +pub use protocol::{Command, ControlMessage, ProtocolError}; +pub use sidecar::{DetectorLoad, IntegrityRecord, RunSidecar, TriggerSource}; +#[cfg(feature = "hardware")] +pub use transport::SerialTransport; +pub use transport::{MockLink, MockTransport, Transport}; +pub use wire::{Frame, FrameHeader, FrameParser, FrameType, ParseEvent, SummaryPayload}; +pub use worker::{IoWorker, WorkerOutput, WorkerRequest}; + +pub mod worker; diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs new file mode 100644 index 0000000..fd7dad0 --- /dev/null +++ b/stage-a-io/src/mock.rs @@ -0,0 +1,283 @@ +//! Mock Stage-A controller for tests and hardware-free plugin development. +//! +//! Implements the v1 command surface (`HELLO`, `STATUS`, `CONFIG`, `ARM`, +//! `RUN`, `START`, `STOP`, `PING`, `FAULT_CLEAR`) with the same idempotency +//! contract as the firmware: replies to recent sequences are cached and +//! resent without re-executing the operation. It can also synthesize +//! photodiode sample/summary frames (sinusoidal drive) so the estimator and +//! plugins can be exercised end to end without a Teensy. + +use std::collections::BTreeMap; + +use crate::protocol::ControlMessage; +use crate::transport::Transport; +use crate::wire::{Frame, FrameHeader, FrameType, SummaryPayload, PROTOCOL_VERSION}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MockState { + SafeIdle, + Configured, + Armed, + Running, +} + +impl MockState { + fn name(self) -> &'static str { + match self { + Self::SafeIdle => "SAFE_IDLE", + Self::Configured => "CONFIGURED", + Self::Armed => "ARMED", + Self::Running => "RUNNING", + } + } +} + +pub struct MockController { + transport: T, + state: MockState, + config: BTreeMap, + config_revision: u32, + reply_cache: Vec<(u32, String)>, + executed_sequences: Vec, + /// Commands executed (used to assert idempotency in tests). + executions: u32, + drop_next_reply: bool, + out_sequence: u32, + line_buffer: Vec, + sample_index: u64, + /// Synthetic optical waveform: codes = center + amplitude*sin(phase). + pub synth_center: f64, + pub synth_amplitude: f64, + pub synth_dark_code: f64, +} + +impl MockController { + pub fn new(transport: T) -> Self { + Self { + transport, + state: MockState::SafeIdle, + config: BTreeMap::new(), + config_revision: 0, + reply_cache: Vec::new(), + executed_sequences: Vec::new(), + executions: 0, + drop_next_reply: false, + out_sequence: 0, + line_buffer: Vec::new(), + sample_index: 0, + synth_center: 2_048.0, + synth_amplitude: 900.0, + synth_dark_code: 40.0, + } + } + + /// Swallow the next reply (simulates a lost USB packet) — the client + /// must retry with the identical sequence. + pub fn drop_first_reply(&mut self) { + self.drop_next_reply = true; + } + + pub fn state(&self) -> MockState { + self.state + } + + /// Serves exactly `n` command lines (counting retries), then returns. + pub fn serve_n_commands(&mut self, n: usize) { + let mut served = 0; + let mut buf = [0_u8; 1024]; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while served < n && std::time::Instant::now() < deadline { + let read = self.transport.read(&mut buf).unwrap_or(0); + if read == 0 { + std::thread::sleep(std::time::Duration::from_millis(1)); + continue; + } + self.line_buffer.extend_from_slice(&buf[..read]); + while let Some(pos) = self.line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = self.line_buffer.drain(..=pos).collect(); + if let Ok(text) = std::str::from_utf8(&line) { + self.handle_line(text.trim_end()); + } + served += 1; + if served >= n { + break; + } + } + } + } + + fn handle_line(&mut self, line: &str) { + let Some(rest) = line.strip_prefix('@') else { + return; + }; + let mut parts = rest.split_ascii_whitespace(); + let Some(sequence) = parts.next().and_then(|s| s.parse::().ok()) else { + return; + }; + // Idempotent retry: replay the cached reply without re-executing. + if let Some((_, cached)) = self + .reply_cache + .iter() + .find(|(cached_seq, _)| *cached_seq == sequence) + { + let payload = cached.clone(); + self.send_control(&payload); + return; + } + assert!( + !self.executed_sequences.contains(&sequence), + "sequence {sequence} re-executed — idempotency broken" + ); + + let verb = parts.next().unwrap_or(""); + let fields: BTreeMap = parts + .filter_map(|part| { + let (key, value) = part.split_once('=')?; + Some((key.to_owned(), value.to_owned())) + }) + .collect(); + + self.executions += 1; + self.executed_sequences.push(sequence); + let reply = self.execute(verb, &fields, sequence); + self.reply_cache.push((sequence, reply.clone())); + if self.reply_cache.len() > 8 { + self.reply_cache.remove(0); + } + if self.drop_next_reply { + self.drop_next_reply = false; + return; + } + self.send_control(&reply); + } + + fn execute(&mut self, verb: &str, fields: &BTreeMap, sequence: u32) -> String { + match verb { + "HELLO" => format!( + "+{sequence} OK protocol=1 firmware=0.1.0-mock board=mock dac_bits=12 \ + capabilities=A1,A2,A3" + ), + "STATUS" => format!( + "+{sequence} OK state={} rev={} executions={}", + self.state.name(), + self.config_revision, + self.executions + ), + "PING" => format!("+{sequence} OK state={}", self.state.name()), + "CONFIG" => { + let mode = fields.get("mode").map(String::as_str).unwrap_or(""); + if !matches!(mode, "A1" | "A2" | "A3") { + return format!("-{sequence} ERR code=BAD_MODE detail=mode"); + } + self.config = fields.clone(); + self.config_revision += 1; + self.state = MockState::Configured; + format!("+{sequence} OK rev={}", self.config_revision) + } + "ARM" => { + if self.state != MockState::Configured { + return format!("-{sequence} ERR code=BAD_STATE detail=arm_requires_config"); + } + self.state = MockState::Armed; + format!("+{sequence} OK state=ARMED rev={}", self.config_revision) + } + "RUN" | "START" => { + if !matches!(self.state, MockState::Armed | MockState::Configured) { + return format!("-{sequence} ERR code=BAD_STATE detail=run_requires_arm"); + } + self.state = MockState::Running; + format!("+{sequence} OK state=RUNNING") + } + "STOP" => { + self.state = MockState::SafeIdle; + format!("+{sequence} OK state=SAFE_IDLE") + } + "FAULT_CLEAR" => format!("+{sequence} OK state={}", self.state.name()), + _ => format!("-{sequence} ERR code=BAD_VERB detail={verb}"), + } + } + + fn send_control(&mut self, payload: &str) { + let frame = self.build_frame(FrameType::Control, payload.as_bytes().to_vec(), 0, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } + + fn build_frame( + &mut self, + frame_type: FrameType, + payload: Vec, + sample_rate_hz: u32, + dropped_samples: u32, + ) -> Frame { + self.out_sequence = self.out_sequence.wrapping_add(1); + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type, + flags: 0, + sequence: self.out_sequence, + payload_bytes: 0, + first_sample_index: self.sample_index, + sample_rate_hz, + dropped_samples, + crc32: 0, + }, + payload, + ) + } + + /// Emits one synthetic sinusoidal sample block (`SamplesU16`). + pub fn emit_sine_block(&mut self, samples: usize, rate_hz: u32, freq_hz: f64) { + let mut payload = Vec::with_capacity(samples * 2); + let mut min_code = u16::MAX; + let mut max_code = 0_u16; + let mut sum = 0_u64; + for i in 0..samples { + let t = (self.sample_index + i as u64) as f64 / f64::from(rate_hz); + let value = self.synth_center + + self.synth_amplitude * (2.0 * std::f64::consts::PI * freq_hz * t).sin(); + let code = value.round().clamp(0.0, 4_095.0) as u16; + min_code = min_code.min(code); + max_code = max_code.max(code); + sum += u64::from(code); + payload.extend_from_slice(&code.to_le_bytes()); + } + let frame = self.build_frame(FrameType::SamplesU16, payload, rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + + let summary = SummaryPayload { + min_code, + max_code, + sample_count: samples as u32, + sum_codes: sum, + first_tick_us: 0, + last_tick_us: ((samples as f64 / f64::from(rate_hz)) * 1e6) as u32, + }; + let frame = self.build_frame(FrameType::Summary, summary.encode(), rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + self.sample_index += samples as u64; + } + + /// Emits a summary frame carrying a nonzero overrun counter. + pub fn emit_summary_with_drops(&mut self, dropped: u32) { + let summary = SummaryPayload { + min_code: 0, + max_code: 0, + sample_count: 0, + sum_codes: 0, + first_tick_us: 0, + last_tick_us: 0, + }; + let frame = self.build_frame(FrameType::Summary, summary.encode(), 20_000, dropped); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } +} + +/// Convenience for tests that need a parsed view of a control payload. +pub fn parse_control(text: &str) -> Option { + ControlMessage::parse(text).ok() +} diff --git a/stage-a-io/src/pdq.rs b/stage-a-io/src/pdq.rs new file mode 100644 index 0000000..0e2119f --- /dev/null +++ b/stage-a-io/src/pdq.rs @@ -0,0 +1,150 @@ +//! `.pdq` writer: preserves every valid PDA1 frame verbatim on disk and +//! tracks run validity. +//! +//! Raw ADC waveforms belong in the PDQ file, never in `HostContext` JSON or +//! per-frame plugin output. A CRC error, frame-sequence gap, or nonzero +//! dropped-sample counter invalidates the run — the file is still written +//! (evidence), but the sidecar must record `valid = false`. + +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use crate::client::StreamIntegrity; +use crate::wire::{crc32, Frame}; + +pub struct PdqWriter { + path: PathBuf, + file: BufWriter, + frames_written: u64, + bytes_written: u64, + running_crc_bytes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PdqSummary { + pub path: PathBuf, + pub frames_written: u64, + pub bytes_written: u64, + /// CRC32 over the whole file contents, recorded in the sidecar. + pub file_crc32: u32, + pub integrity: StreamIntegrity, + pub valid: bool, +} + +impl PdqWriter { + pub fn create(path: impl AsRef) -> std::io::Result { + let path = path.as_ref().to_owned(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + Ok(Self { + file: BufWriter::new(File::create(&path)?), + path, + frames_written: 0, + bytes_written: 0, + running_crc_bytes: Vec::new(), + }) + } + + pub fn write_frame(&mut self, frame: &Frame) -> std::io::Result<()> { + let bytes = frame.to_bytes(); + self.file.write_all(&bytes)?; + self.frames_written += 1; + self.bytes_written += bytes.len() as u64; + self.running_crc_bytes.extend_from_slice(&bytes); + Ok(()) + } + + /// Flushes and closes the file, returning the summary for the sidecar. + pub fn finish(mut self, integrity: StreamIntegrity) -> std::io::Result { + self.file.flush()?; + Ok(PdqSummary { + file_crc32: crc32(&self.running_crc_bytes), + path: self.path, + frames_written: self.frames_written, + bytes_written: self.bytes_written, + valid: integrity.is_clean(), + integrity, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::wire::{FrameHeader, FrameType, PROTOCOL_VERSION}; + + fn frame(sequence: u32) -> Frame { + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Control, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 0, + dropped_samples: 0, + crc32: 0, + }, + format!("+{sequence} OK").into_bytes(), + ) + } + + #[test] + fn writes_frames_verbatim_and_reports_validity() { + let dir = std::env::temp_dir().join(format!( + "stage-a-io-pdq-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let path = dir.join("run.pdq"); + + let mut writer = PdqWriter::create(&path).expect("create pdq"); + let first = frame(1); + let second = frame(2); + writer.write_frame(&first).expect("write"); + writer.write_frame(&second).expect("write"); + let summary = writer + .finish(StreamIntegrity::default()) + .expect("finish pdq"); + + assert!(summary.valid); + assert_eq!(summary.frames_written, 2); + let on_disk = std::fs::read(&path).expect("read back"); + let mut expected = first.to_bytes(); + expected.extend_from_slice(&second.to_bytes()); + assert_eq!(on_disk, expected); + assert_eq!(summary.file_crc32, crate::wire::crc32(&expected)); + + std::fs::remove_dir_all(dir).expect("cleanup"); + } + + #[test] + fn integrity_faults_invalidate_the_run_but_keep_the_file() { + let dir = std::env::temp_dir().join(format!( + "stage-a-io-pdq-invalid-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let path = dir.join("run.pdq"); + + let mut writer = PdqWriter::create(&path).expect("create pdq"); + writer.write_frame(&frame(1)).expect("write"); + let summary = writer + .finish(StreamIntegrity { + dropped_samples: 5, + ..StreamIntegrity::default() + }) + .expect("finish pdq"); + + assert!(!summary.valid); + assert!(path.exists(), "evidence file is preserved"); + std::fs::remove_dir_all(dir).expect("cleanup"); + } +} diff --git a/stage-a-io/src/protocol.rs b/stage-a-io/src/protocol.rs new file mode 100644 index 0000000..254130f --- /dev/null +++ b/stage-a-io/src/protocol.rs @@ -0,0 +1,226 @@ +//! ASCII command / control-reply grammar (host → Teensy and CONTROL frame +//! payloads), per the Stage-A serial protocol v1: +//! +//! ```text +//! Host request: @ key=value key=value\n +//! CONTROL reply payload: + OK key=value ... +//! CONTROL reply payload: - ERR code= detail= +//! Async CONTROL payload: ! key=value ... +//! ``` +//! +//! Commands are printable ASCII, max 192 bytes, integer values only. + +use std::collections::BTreeMap; +use std::fmt; + +pub const MAX_COMMAND_BYTES: usize = 192; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Command { + pub verb: String, + /// Ordered key=value fields (insertion order is preserved on the wire; + /// a BTreeMap would silently reorder, so use a Vec of pairs). + pub fields: Vec<(String, String)>, +} + +impl Command { + pub fn new(verb: &str) -> Self { + Self { + verb: verb.to_owned(), + fields: Vec::new(), + } + } + + pub fn field(mut self, key: &str, value: impl fmt::Display) -> Self { + self.fields.push((key.to_owned(), value.to_string())); + self + } + + /// Encodes `@ VERB k=v ...\n`, validating the printable-ASCII and + /// length constraints. + pub fn encode(&self, sequence: u32) -> Result, ProtocolError> { + let mut line = format!("@{sequence} {}", self.verb); + for (key, value) in &self.fields { + line.push(' '); + line.push_str(key); + line.push('='); + line.push_str(value); + } + line.push('\n'); + if line.len() > MAX_COMMAND_BYTES { + return Err(ProtocolError::CommandTooLong(line.len())); + } + if !line + .bytes() + .all(|b| b == b'\n' || (0x20..=0x7E).contains(&b)) + { + return Err(ProtocolError::NonPrintable); + } + Ok(line.into_bytes()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ControlMessage { + /// `+ OK key=value ...` + Ok { + sequence: u32, + fields: BTreeMap, + }, + /// `- ERR code= detail=` + Err { + sequence: u32, + code: String, + detail: String, + }, + /// `! key=value ...` + Async { + name: String, + fields: BTreeMap, + }, +} + +impl ControlMessage { + pub fn parse(text: &str) -> Result { + let text = text.trim_end_matches(['\r', '\n']); + let mut parts = text.split_ascii_whitespace(); + let head = parts.next().ok_or(ProtocolError::EmptyControl)?; + match head.as_bytes().first() { + Some(b'+') => { + let sequence = head[1..] + .parse() + .map_err(|_| ProtocolError::BadSequence(head.to_owned()))?; + let ok = parts.next(); + if ok != Some("OK") { + return Err(ProtocolError::Malformed(text.to_owned())); + } + Ok(Self::Ok { + sequence, + fields: parse_fields(parts), + }) + } + Some(b'-') => { + let sequence = head[1..] + .parse() + .map_err(|_| ProtocolError::BadSequence(head.to_owned()))?; + let err = parts.next(); + if err != Some("ERR") { + return Err(ProtocolError::Malformed(text.to_owned())); + } + let fields = parse_fields(parts); + Ok(Self::Err { + sequence, + code: fields.get("code").cloned().unwrap_or_default(), + detail: fields.get("detail").cloned().unwrap_or_default(), + }) + } + Some(b'!') => Ok(Self::Async { + name: head[1..].to_owned(), + fields: parse_fields(parts), + }), + _ => Err(ProtocolError::Malformed(text.to_owned())), + } + } + + pub fn sequence(&self) -> Option { + match self { + Self::Ok { sequence, .. } | Self::Err { sequence, .. } => Some(*sequence), + Self::Async { .. } => None, + } + } +} + +fn parse_fields<'a>(parts: impl Iterator) -> BTreeMap { + parts + .filter_map(|part| { + let (key, value) = part.split_once('=')?; + Some((key.to_owned(), value.to_owned())) + }) + .collect() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProtocolError { + CommandTooLong(usize), + NonPrintable, + EmptyControl, + BadSequence(String), + Malformed(String), +} + +impl fmt::Display for ProtocolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CommandTooLong(len) => { + write!(f, "command is {len} bytes (max {MAX_COMMAND_BYTES})") + } + Self::NonPrintable => f.write_str("command contains non-printable bytes"), + Self::EmptyControl => f.write_str("empty control payload"), + Self::BadSequence(head) => write!(f, "unparseable sequence in {head:?}"), + Self::Malformed(text) => write!(f, "malformed control payload {text:?}"), + } + } +} + +impl std::error::Error for ProtocolError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodes_commands_with_ordered_fields() { + let cmd = Command::new("CONFIG") + .field("mode", "A1") + .field("freq_mhz", 12_500) + .field("center_dac", 2_048) + .field("amplitude_dac", 512); + assert_eq!( + String::from_utf8(cmd.encode(3).expect("encodes")).unwrap(), + "@3 CONFIG mode=A1 freq_mhz=12500 center_dac=2048 amplitude_dac=512\n" + ); + } + + #[test] + fn rejects_oversized_and_non_printable_commands() { + let long = Command::new("X").field("k", "y".repeat(200)); + assert!(matches!( + long.encode(1), + Err(ProtocolError::CommandTooLong(_)) + )); + let bad = Command::new("X").field("k", "\u{7f}"); + assert!(matches!(bad.encode(1), Err(ProtocolError::NonPrintable))); + } + + #[test] + fn parses_ok_err_and_async_payloads() { + let ok = ControlMessage::parse("+12 OK state=ARMED rev=4").expect("ok parses"); + match ok { + ControlMessage::Ok { sequence, fields } => { + assert_eq!(sequence, 12); + assert_eq!(fields.get("rev").map(String::as_str), Some("4")); + } + other => panic!("unexpected {other:?}"), + } + + let err = + ControlMessage::parse("-13 ERR code=BOUNDS detail=amplitude_dac").expect("err parses"); + assert_eq!( + err, + ControlMessage::Err { + sequence: 13, + code: "BOUNDS".into(), + detail: "amplitude_dac".into() + } + ); + + let async_msg = ControlMessage::parse("!APPLIED rev=4").expect("async parses"); + match async_msg { + ControlMessage::Async { name, fields } => { + assert_eq!(name, "APPLIED"); + assert_eq!(fields.get("rev").map(String::as_str), Some("4")); + } + other => panic!("unexpected {other:?}"), + } + } +} diff --git a/stage-a-io/src/sidecar.rs b/stage-a-io/src/sidecar.rs new file mode 100644 index 0000000..0b591bc --- /dev/null +++ b/stage-a-io/src/sidecar.rs @@ -0,0 +1,216 @@ +//! Run sidecar (manifest): everything needed to reproduce or audit one +//! Stage-A recording, written as JSON next to the camera RAW / PDQ files. +//! +//! Per the control-software spec, each recording sidecar includes the run +//! ID, plugin/firmware/protocol versions, raw PDQ path and checksum, +//! ADC/front-end calibration, load, configured and measured sample cadence, +//! drop/CRC counters, the ACKed configuration revision, bias set, optical +//! configuration, flux point, measured `a`, and trigger source. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::client::StreamIntegrity; +use crate::estimator::{AdcCalibration, ContrastEstimate}; +use crate::pdq::PdqSummary; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TriggerSource { + /// Teensy waveform phase-0 sync TTL (A1/A3 drive fiducial). + DrivePhase0, + /// Photodiode → comparator 50 % crossing (A2 light fiducial). + Comparator, + /// No hardware trigger wired; software phase recovery in use. + None, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DetectorLoad { + /// 50 Ω — A1/A2 (speed over signal). + FiftyOhm, + /// Characterised high-Z load — A3 only ($f \ll f_c$). + HighZ { nominal_ohms: u64 }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RunSidecar { + pub run_id: String, + pub protocol: String, + pub created_utc: String, + + pub plugin_name: String, + pub plugin_version: String, + pub firmware_version: String, + pub wire_protocol_version: u8, + + /// Path + CRC32 of the raw PDQ photodiode file. + pub pdq_path: PathBuf, + pub pdq_crc32: u32, + pub pdq_frames: u64, + /// Path of the camera RAW recording this run belongs to, if any. + pub camera_raw_path: Option, + + pub adc_calibration: AdcCalibration, + pub detector_load: DetectorLoad, + pub configured_sample_rate_hz: u32, + pub measured_sample_rate_hz: Option, + + pub integrity: IntegrityRecord, + /// Overall validity — false on any drop/CRC/sequence/cadence fault or + /// estimator rejection. An invalid point is re-measured, never patched. + pub valid: bool, + + /// ACKed controller configuration (verbatim key=value fields) and its + /// revision, exactly as the firmware confirmed them. + pub acked_config_revision: Option, + pub acked_config: BTreeMap, + + /// Frozen camera bias set identifier (registry lives in the knowledge + /// base `setup/bias-sets.md`). + pub bias_set: Option, + /// Optical configuration / flux point labels from the run plan. + pub optical_configuration: Option, + pub flux_point: Option, + + /// Measured optical log-contrast for this run/point, when applicable. + pub measured_contrast: Option, + pub trigger_source: TriggerSource, + + /// Free-form notes (operator observations, deviations). + pub notes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct IntegrityRecord { + pub skipped_bytes: u64, + pub crc_failures: u64, + pub sequence_gaps: u64, + pub dropped_samples: u64, +} + +impl From for IntegrityRecord { + fn from(value: StreamIntegrity) -> Self { + Self { + skipped_bytes: value.skipped_bytes, + crc_failures: value.crc_failures, + sequence_gaps: value.sequence_gaps, + dropped_samples: value.dropped_samples, + } + } +} + +impl RunSidecar { + /// Builds a sidecar skeleton from a finished PDQ file. Protocol fields + /// and run metadata are filled by the owning plugin before writing. + pub fn from_pdq(run_id: &str, protocol: &str, pdq: &PdqSummary) -> Self { + Self { + run_id: run_id.to_owned(), + protocol: protocol.to_owned(), + created_utc: now_utc_iso8601(), + plugin_name: String::new(), + plugin_version: String::new(), + firmware_version: String::new(), + wire_protocol_version: crate::wire::PROTOCOL_VERSION, + pdq_path: pdq.path.clone(), + pdq_crc32: pdq.file_crc32, + pdq_frames: pdq.frames_written, + camera_raw_path: None, + adc_calibration: AdcCalibration::default(), + detector_load: DetectorLoad::FiftyOhm, + configured_sample_rate_hz: 0, + measured_sample_rate_hz: None, + integrity: pdq.integrity.into(), + valid: pdq.valid, + acked_config_revision: None, + acked_config: BTreeMap::new(), + bias_set: None, + optical_configuration: None, + flux_point: None, + measured_contrast: None, + trigger_source: TriggerSource::None, + notes: Vec::new(), + } + } + + pub fn write_json(&self, path: impl AsRef) -> std::io::Result<()> { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let json = serde_json::to_vec_pretty(self)?; + std::fs::write(path, json) + } + + pub fn read_json(path: impl AsRef) -> std::io::Result { + let bytes = std::fs::read(path)?; + serde_json::from_slice(&bytes).map_err(std::io::Error::other) + } +} + +fn now_utc_iso8601() -> String { + // Seconds-resolution UTC timestamp without pulling in chrono. + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = secs / 86_400; + let (year, month, day) = civil_from_days(days as i64); + let rem = secs % 86_400; + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", + rem / 3_600, + (rem % 3_600) / 60, + rem % 60 + ) +} + +/// Howard Hinnant's `civil_from_days` (public domain algorithm). +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = (z - era * 146_097) as u64; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + (if m <= 2 { y + 1 } else { y }, m, d) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sidecar_round_trips_through_json() { + let pdq = PdqSummary { + path: PathBuf::from("/data/A1-20260713-01.pdq"), + frames_written: 128, + bytes_written: 65_536, + file_crc32: 0xDEAD_BEEF, + integrity: StreamIntegrity::default(), + valid: true, + }; + let mut sidecar = RunSidecar::from_pdq("A1-20260713-01", "A1", &pdq); + sidecar.plugin_name = "stage-a-a1".into(); + sidecar.acked_config_revision = Some(4); + sidecar.acked_config.insert("mode".into(), "A1".into()); + sidecar.trigger_source = TriggerSource::DrivePhase0; + + let json = serde_json::to_string(&sidecar).expect("serializes"); + let decoded: RunSidecar = serde_json::from_str(&json).expect("deserializes"); + assert_eq!(decoded, sidecar); + assert!(decoded.created_utc.ends_with('Z')); + } + + #[test] + fn civil_from_days_matches_known_dates() { + assert_eq!(civil_from_days(0), (1970, 1, 1)); + assert_eq!(civil_from_days(20_282), (2025, 7, 13)); + } +} diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs new file mode 100644 index 0000000..27dea22 --- /dev/null +++ b/stage-a-io/src/transport.rs @@ -0,0 +1,113 @@ +//! Byte transports: the real USB serial port and an in-memory mock. +//! +//! Exactly one armed plugin owns the port at a time; opening a busy device +//! is a visible error, never a silent second connection (the OS enforces +//! exclusivity via `serialport`'s exclusive open on POSIX). + +use std::io; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +pub trait Transport: Send { + /// Reads whatever is available into `buf`, blocking up to the + /// transport's timeout. `Ok(0)` means "nothing arrived this poll". + fn read(&mut self, buf: &mut [u8]) -> io::Result; + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()>; +} + +/// Real serial port. Construction fails visibly if the device is busy or +/// absent. +#[cfg(feature = "hardware")] +pub struct SerialTransport { + port: Box, +} + +#[cfg(feature = "hardware")] +impl SerialTransport { + pub fn open(path: &str, baud: u32, poll_timeout: Duration) -> io::Result { + let port = serialport::new(path, baud) + .timeout(poll_timeout) + .open() + .map_err(|err| io::Error::other(format!("opening {path} failed: {err}")))?; + Ok(Self { port }) + } +} + +#[cfg(feature = "hardware")] +impl Transport for SerialTransport { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self.port.read(buf) { + Ok(n) => Ok(n), + Err(err) if err.kind() == io::ErrorKind::TimedOut => Ok(0), + Err(err) => Err(err), + } + } + + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { + io::Write::write_all(&mut self.port, bytes) + } +} + +/// Shared in-memory duplex used by tests and the mock controller: the +/// "host" side reads what the "device" side wrote and vice versa. +#[derive(Default)] +struct DuplexState { + to_host: Vec, + to_device: Vec, +} + +#[derive(Clone, Default)] +pub struct MockLink { + state: Arc>, +} + +impl MockLink { + pub fn new() -> Self { + Self::default() + } + + pub fn host_end(&self) -> MockTransport { + MockTransport { + state: Arc::clone(&self.state), + is_host: true, + } + } + + pub fn device_end(&self) -> MockTransport { + MockTransport { + state: Arc::clone(&self.state), + is_host: false, + } + } +} + +pub struct MockTransport { + state: Arc>, + is_host: bool, +} + +impl Transport for MockTransport { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let source = if self.is_host { + &mut state.to_host + } else { + &mut state.to_device + }; + let n = source.len().min(buf.len()); + buf[..n].copy_from_slice(&source[..n]); + source.drain(..n); + Ok(n) + } + + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { + let mut state = self.state.lock().unwrap_or_else(|p| p.into_inner()); + let sink = if self.is_host { + &mut state.to_device + } else { + &mut state.to_host + }; + sink.extend_from_slice(bytes); + Ok(()) + } +} diff --git a/stage-a-io/src/wire.rs b/stage-a-io/src/wire.rs new file mode 100644 index 0000000..33e8caa --- /dev/null +++ b/stage-a-io/src/wire.rs @@ -0,0 +1,438 @@ +//! PDA1 binary wire format (Teensy → host). +//! +//! Mirrors `stage-a-controller/include/wire_protocol.h` exactly: a packed +//! 36-byte little-endian header followed by `payload_bytes` of payload, +//! integrity-protected by CRC32 (IEEE, reflected) over the zeroed-CRC header +//! plus payload. The host must tolerate arbitrary USB fragmentation and +//! resynchronise at the next valid magic + CRC. + +/// `"PDA1"` interpreted as a little-endian `u32`. +pub const MAGIC: u32 = 0x3141_4450; +pub const PROTOCOL_VERSION: u8 = 1; +pub const HEADER_BYTES: usize = 36; + +/// Maximum payload the parser will attempt to buffer. Larger claimed sizes +/// are treated as corruption and trigger resynchronisation. +pub const MAX_PAYLOAD_BYTES: usize = 1 << 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameType { + Control, + SamplesU16, + Summary, + Marker, + Unknown(u8), +} + +impl FrameType { + pub fn from_raw(raw: u8) -> Self { + match raw { + 1 => Self::Control, + 2 => Self::SamplesU16, + 3 => Self::Summary, + 4 => Self::Marker, + other => Self::Unknown(other), + } + } + + pub fn to_raw(self) -> u8 { + match self { + Self::Control => 1, + Self::SamplesU16 => 2, + Self::Summary => 3, + Self::Marker => 4, + Self::Unknown(other) => other, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameHeader { + pub version: u8, + pub frame_type: FrameType, + pub flags: u16, + pub sequence: u32, + pub payload_bytes: u32, + pub first_sample_index: u64, + pub sample_rate_hz: u32, + pub dropped_samples: u32, + pub crc32: u32, +} + +impl FrameHeader { + pub fn parse(bytes: &[u8; HEADER_BYTES]) -> Option { + let magic = u32::from_le_bytes(bytes[0..4].try_into().ok()?); + if magic != MAGIC { + return None; + } + Some(Self { + version: bytes[4], + frame_type: FrameType::from_raw(bytes[5]), + flags: u16::from_le_bytes(bytes[6..8].try_into().ok()?), + sequence: u32::from_le_bytes(bytes[8..12].try_into().ok()?), + payload_bytes: u32::from_le_bytes(bytes[12..16].try_into().ok()?), + first_sample_index: u64::from_le_bytes(bytes[16..24].try_into().ok()?), + sample_rate_hz: u32::from_le_bytes(bytes[24..28].try_into().ok()?), + dropped_samples: u32::from_le_bytes(bytes[28..32].try_into().ok()?), + crc32: u32::from_le_bytes(bytes[32..36].try_into().ok()?), + }) + } + + pub fn encode(&self) -> [u8; HEADER_BYTES] { + let mut out = [0_u8; HEADER_BYTES]; + out[0..4].copy_from_slice(&MAGIC.to_le_bytes()); + out[4] = self.version; + out[5] = self.frame_type.to_raw(); + out[6..8].copy_from_slice(&self.flags.to_le_bytes()); + out[8..12].copy_from_slice(&self.sequence.to_le_bytes()); + out[12..16].copy_from_slice(&self.payload_bytes.to_le_bytes()); + out[16..24].copy_from_slice(&self.first_sample_index.to_le_bytes()); + out[24..28].copy_from_slice(&self.sample_rate_hz.to_le_bytes()); + out[28..32].copy_from_slice(&self.dropped_samples.to_le_bytes()); + out[32..36].copy_from_slice(&self.crc32.to_le_bytes()); + out + } +} + +/// One complete, CRC-verified frame. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + pub header: FrameHeader, + pub payload: Vec, +} + +impl Frame { + /// Builds a frame with a freshly computed CRC (mock/firmware side). + pub fn build(mut header: FrameHeader, payload: Vec) -> Self { + header.payload_bytes = payload.len() as u32; + header.crc32 = frame_crc(&header, &payload); + Self { header, payload } + } + + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(HEADER_BYTES + self.payload.len()); + out.extend_from_slice(&self.header.encode()); + out.extend_from_slice(&self.payload); + out + } + + /// Decodes the payload of a `Summary` frame. + pub fn summary(&self) -> Option { + if self.header.frame_type != FrameType::Summary || self.payload.len() != 24 { + return None; + } + let p = &self.payload; + Some(SummaryPayload { + min_code: u16::from_le_bytes(p[0..2].try_into().ok()?), + max_code: u16::from_le_bytes(p[2..4].try_into().ok()?), + sample_count: u32::from_le_bytes(p[4..8].try_into().ok()?), + sum_codes: u64::from_le_bytes(p[8..16].try_into().ok()?), + first_tick_us: u32::from_le_bytes(p[16..20].try_into().ok()?), + last_tick_us: u32::from_le_bytes(p[20..24].try_into().ok()?), + }) + } + + /// Decodes the payload of a `SamplesU16` frame into ADC codes. + pub fn samples(&self) -> Option> { + if self.header.frame_type != FrameType::SamplesU16 || self.payload.len() % 2 != 0 { + return None; + } + Some( + self.payload + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + ) + } + + /// The ASCII payload of a `Control` frame. + pub fn control_text(&self) -> Option<&str> { + if self.header.frame_type != FrameType::Control { + return None; + } + std::str::from_utf8(&self.payload).ok() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SummaryPayload { + pub min_code: u16, + pub max_code: u16, + pub sample_count: u32, + pub sum_codes: u64, + pub first_tick_us: u32, + pub last_tick_us: u32, +} + +impl SummaryPayload { + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(24); + out.extend_from_slice(&self.min_code.to_le_bytes()); + out.extend_from_slice(&self.max_code.to_le_bytes()); + out.extend_from_slice(&self.sample_count.to_le_bytes()); + out.extend_from_slice(&self.sum_codes.to_le_bytes()); + out.extend_from_slice(&self.first_tick_us.to_le_bytes()); + out.extend_from_slice(&self.last_tick_us.to_le_bytes()); + out + } + + pub fn mean_code(&self) -> f64 { + if self.sample_count == 0 { + return 0.0; + } + self.sum_codes as f64 / f64::from(self.sample_count) + } +} + +/// CRC32 (IEEE, reflected, init/final 0xFFFF_FFFF) — identical to the +/// firmware's `crc32Update` loop. +pub fn crc32(data: &[u8]) -> u32 { + crc32_update(0xFFFF_FFFF, data) ^ 0xFFFF_FFFF +} + +fn crc32_update(mut crc: u32, data: &[u8]) -> u32 { + for &byte in data { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = 0_u32.wrapping_sub(crc & 1); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + crc +} + +/// CRC over the zeroed-CRC header plus payload (firmware `frameCrc`). +pub fn frame_crc(header: &FrameHeader, payload: &[u8]) -> u32 { + let mut zeroed = *header; + zeroed.crc32 = 0; + let mut crc = 0xFFFF_FFFF_u32; + crc = crc32_update(crc, &zeroed.encode()); + crc = crc32_update(crc, payload); + crc ^ 0xFFFF_FFFF +} + +/// What the incremental parser reports for each recovered unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ParseEvent { + Frame(Frame), + /// Bytes were skipped or a frame failed its CRC — the stream stays + /// usable, but the run must be flagged invalid. + Corruption { + skipped_bytes: usize, + crc_failures: usize, + }, +} + +/// Incremental PDA1 parser tolerating arbitrary fragmentation. +/// +/// Feed raw serial bytes with [`FrameParser::extend`], then drain complete +/// frames with [`FrameParser::next_event`]. On a bad magic the parser skips +/// forward one byte at a time; on a bad CRC it discards the candidate header +/// and rescans from the next byte, so a corrupted stream re-locks at the +/// next genuine frame boundary. +#[derive(Debug, Default)] +pub struct FrameParser { + buffer: Vec, + skipped_bytes: usize, + crc_failures: usize, +} + +impl FrameParser { + pub fn extend(&mut self, bytes: &[u8]) { + self.buffer.extend_from_slice(bytes); + } + + pub fn next_event(&mut self) -> Option { + loop { + // Scan to the next plausible magic. + let mut offset = 0; + while self.buffer.len() >= offset + 4 + && u32::from_le_bytes(self.buffer[offset..offset + 4].try_into().unwrap()) != MAGIC + { + offset += 1; + } + if offset > 0 { + self.buffer.drain(..offset); + self.skipped_bytes += offset; + } + + if self.buffer.len() < HEADER_BYTES { + return self.take_corruption(); + } + + let header_bytes: [u8; HEADER_BYTES] = self.buffer[..HEADER_BYTES].try_into().unwrap(); + let Some(header) = FrameHeader::parse(&header_bytes) else { + // Magic matched but parse failed (cannot happen today, but + // stay defensive): skip one byte and rescan. + self.buffer.drain(..1); + self.skipped_bytes += 1; + continue; + }; + + let payload_bytes = header.payload_bytes as usize; + if payload_bytes > MAX_PAYLOAD_BYTES { + self.buffer.drain(..1); + self.skipped_bytes += 1; + continue; + } + if self.buffer.len() < HEADER_BYTES + payload_bytes { + // Wait for more bytes; report any corruption noticed so far. + return self.take_corruption(); + } + + let payload = self.buffer[HEADER_BYTES..HEADER_BYTES + payload_bytes].to_vec(); + if frame_crc(&header, &payload) != header.crc32 { + self.crc_failures += 1; + self.buffer.drain(..1); + self.skipped_bytes += 1; + continue; + } + + self.buffer.drain(..HEADER_BYTES + payload_bytes); + if let Some(corruption) = self.take_corruption() { + // Deliver the corruption notice first; the verified frame is + // still buffered as raw bytes, so re-parse it next call. + let frame = Frame { header, payload }; + let mut bytes = frame.to_bytes(); + bytes.extend_from_slice(&self.buffer); + self.buffer = bytes; + return Some(corruption); + } + return Some(ParseEvent::Frame(Frame { header, payload })); + } + } + + fn take_corruption(&mut self) -> Option { + if self.skipped_bytes == 0 && self.crc_failures == 0 { + return None; + } + let event = ParseEvent::Corruption { + skipped_bytes: self.skipped_bytes, + crc_failures: self.crc_failures, + }; + self.skipped_bytes = 0; + self.crc_failures = 0; + Some(event) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn control_frame(sequence: u32, text: &str) -> Frame { + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Control, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 0, + dropped_samples: 0, + crc32: 0, + }, + text.as_bytes().to_vec(), + ) + } + + #[test] + fn round_trips_a_frame_through_arbitrary_fragmentation() { + let frame = control_frame(7, "+7 OK state=SAFE_IDLE"); + let bytes = frame.to_bytes(); + + let mut parser = FrameParser::default(); + for chunk in bytes.chunks(3) { + parser.extend(chunk); + } + assert_eq!(parser.next_event(), Some(ParseEvent::Frame(frame))); + assert_eq!(parser.next_event(), None); + } + + #[test] + fn resynchronises_after_garbage_and_reports_corruption() { + let frame = control_frame(1, "+1 OK"); + let mut bytes = b"garbage!".to_vec(); + bytes.extend_from_slice(&frame.to_bytes()); + + let mut parser = FrameParser::default(); + parser.extend(&bytes); + assert_eq!( + parser.next_event(), + Some(ParseEvent::Corruption { + skipped_bytes: 8, + crc_failures: 0 + }) + ); + assert_eq!(parser.next_event(), Some(ParseEvent::Frame(frame))); + } + + #[test] + fn detects_crc_corruption_and_relocks_on_next_frame() { + let bad = control_frame(1, "+1 OK"); + let good = control_frame(2, "!STATUS state=RUNNING"); + let mut bytes = bad.to_bytes(); + let len = bytes.len(); + bytes[len - 1] ^= 0xFF; // corrupt payload -> CRC mismatch + bytes.extend_from_slice(&good.to_bytes()); + + let mut parser = FrameParser::default(); + parser.extend(&bytes); + let corruption = parser.next_event(); + match corruption { + Some(ParseEvent::Corruption { crc_failures, .. }) => assert!(crc_failures >= 1), + other => panic!("expected corruption, got {other:?}"), + } + assert_eq!(parser.next_event(), Some(ParseEvent::Frame(good))); + } + + #[test] + fn summary_payload_round_trips() { + let summary = SummaryPayload { + min_code: 12, + max_code: 3_900, + sample_count: 256, + sum_codes: 500_000, + first_tick_us: 1_000, + last_tick_us: 13_800, + }; + let frame = Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Summary, + flags: 0, + sequence: 5, + payload_bytes: 0, + first_sample_index: 4_096, + sample_rate_hz: 20_000, + dropped_samples: 0, + crc32: 0, + }, + summary.encode(), + ); + assert_eq!(frame.summary(), Some(summary)); + assert!((summary.mean_code() - 1953.125).abs() < 1e-9); + } + + #[test] + fn samples_frame_decodes_codes() { + let codes = [1_u16, 2, 4_095]; + let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); + let frame = Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::SamplesU16, + flags: 0, + sequence: 9, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 20_000, + dropped_samples: 0, + crc32: 0, + }, + payload, + ); + assert_eq!(frame.samples(), Some(codes.to_vec())); + } +} diff --git a/stage-a-io/src/worker.rs b/stage-a-io/src/worker.rs new file mode 100644 index 0000000..57ffaee --- /dev/null +++ b/stage-a-io/src/worker.rs @@ -0,0 +1,229 @@ +//! Bounded background I/O worker. +//! +//! The owning plugin's `process_frame()` must never block on serial: it only +//! drains this worker's bounded output queue and pushes bounded requests. +//! The worker thread owns the [`StageAClient`] (and thereby the serial +//! port), sends `PING` at 2 Hz while the controller is armed/running, and +//! requests `STOP` on shutdown. Firmware safety does not depend on that +//! STOP arriving — the on-device watchdog falls back to `SAFE_IDLE` — but a +//! clean stop is always attempted. + +use std::collections::BTreeMap; +use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use crate::client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; +use crate::protocol::Command; +use crate::transport::Transport; + +pub const COMMAND_QUEUE_DEPTH: usize = 16; +pub const OUTPUT_QUEUE_DEPTH: usize = 256; +const PING_INTERVAL: Duration = Duration::from_millis(500); +const IDLE_POLL: Duration = Duration::from_millis(5); + +/// Requests the plugin can queue for the worker. +#[derive(Debug, Clone)] +pub enum WorkerRequest { + /// Send a command and report its reply (or error) as a `Reply` output. + Send { tag: u64, command: Command }, + /// Enable/disable the 2 Hz watchdog ping (armed/running phases). + SetPinging(bool), + /// Stop the controller and shut the worker down. + Shutdown { reason: String }, +} + +/// Bounded outputs the plugin drains from `process_frame()`. +#[derive(Debug)] +pub enum WorkerOutput { + Reply { + tag: u64, + result: Result, String>, + }, + Event(DeviceEvent), + Integrity(StreamIntegrity), + /// The worker exited (clean shutdown or transport failure). + Stopped { + reason: String, + }, +} + +pub struct IoWorker { + requests: SyncSender, + outputs: Receiver, + join: Option>, +} + +impl IoWorker { + /// Spawns the worker over an already-open transport. Opening the + /// transport (and failing visibly if the device is busy) is the + /// caller's responsibility, in `LiveCapture` with effects allowed only. + pub fn spawn(client: StageAClient) -> Self { + let (request_tx, request_rx) = std::sync::mpsc::sync_channel(COMMAND_QUEUE_DEPTH); + let (output_tx, output_rx) = std::sync::mpsc::sync_channel(OUTPUT_QUEUE_DEPTH); + let join = std::thread::Builder::new() + .name("stage-a-io".into()) + .spawn(move || run_worker(client, request_rx, output_tx)) + .expect("spawning the stage-a I/O thread must succeed"); + Self { + requests: request_tx, + outputs: output_rx, + join: Some(join), + } + } + + /// Non-blocking enqueue; a full queue is a visible error, not a stall. + pub fn try_send(&self, request: WorkerRequest) -> Result<(), String> { + self.requests.try_send(request).map_err(|err| match err { + TrySendError::Full(_) => "stage-a I/O command queue is full".to_owned(), + TrySendError::Disconnected(_) => "stage-a I/O worker is gone".to_owned(), + }) + } + + /// Drains everything currently queued, without blocking. + pub fn drain_outputs(&self) -> Vec { + let mut out = Vec::new(); + while let Ok(output) = self.outputs.try_recv() { + out.push(output); + } + out + } + + /// Requests a controller STOP and joins the worker. + pub fn shutdown(mut self, reason: &str) { + let _ = self.requests.try_send(WorkerRequest::Shutdown { + reason: reason.to_owned(), + }); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +impl Drop for IoWorker { + fn drop(&mut self) { + let _ = self.requests.try_send(WorkerRequest::Shutdown { + reason: "worker dropped".to_owned(), + }); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +fn run_worker( + mut client: StageAClient, + requests: Receiver, + outputs: SyncSender, +) { + let mut pinging = false; + let mut last_ping = Instant::now(); + let mut last_integrity = client.integrity(); + + let stop_reason = loop { + match requests.recv_timeout(IDLE_POLL) { + Ok(WorkerRequest::Send { tag, command }) => { + let result = client + .request(&command) + .map_err(|err: ClientError| err.to_string()); + if outputs + .try_send(WorkerOutput::Reply { tag, result }) + .is_err() + { + break "output queue closed".to_owned(); + } + } + Ok(WorkerRequest::SetPinging(enabled)) => { + pinging = enabled; + last_ping = Instant::now(); + } + Ok(WorkerRequest::Shutdown { reason }) => break reason, + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break "request queue closed".to_owned(), + } + + match client.poll_events() { + Ok(events) => { + for event in events { + // Bounded best-effort delivery: a full output queue drops + // live telemetry, never blocks the serial loop. Exact + // data is preserved by the PDQ writer downstream of the + // worker owner, which uses Reply-driven flow instead. + let _ = outputs.try_send(WorkerOutput::Event(event)); + } + } + Err(err) => { + let _ = outputs.try_send(WorkerOutput::Reply { + tag: 0, + result: Err(err.to_string()), + }); + break "transport failure".to_owned(); + } + } + + let integrity = client.integrity(); + if integrity != last_integrity { + last_integrity = integrity; + let _ = outputs.try_send(WorkerOutput::Integrity(integrity)); + } + + if pinging && last_ping.elapsed() >= PING_INTERVAL { + last_ping = Instant::now(); + let _ = client.request(&Command::new("PING")); + } + }; + + // Best-effort clean stop; the firmware watchdog is the real guarantee. + let _ = client.request(&Command::new("STOP").field("reason", stop_reason.replace(' ', "_"))); + let _ = outputs.try_send(WorkerOutput::Stopped { + reason: stop_reason, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::MockController; + use crate::transport::MockLink; + + #[test] + fn worker_round_trips_commands_and_stops_cleanly() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + let worker = IoWorker::spawn(client); + + // HELLO via the worker, served by the mock on this thread. + worker + .try_send(WorkerRequest::Send { + tag: 1, + command: Command::new("HELLO").field("protocol", 1), + }) + .expect("enqueue"); + controller.serve_n_commands(1); + + let deadline = Instant::now() + Duration::from_secs(1); + let mut reply_seen = false; + while Instant::now() < deadline && !reply_seen { + for output in worker.drain_outputs() { + if let WorkerOutput::Reply { tag: 1, result } = output { + let fields = result.expect("HELLO succeeds"); + assert_eq!(fields.get("protocol").map(String::as_str), Some("1")); + reply_seen = true; + } + } + std::thread::sleep(Duration::from_millis(2)); + } + assert!(reply_seen, "HELLO reply must reach the plugin queue"); + + // Shutdown must send STOP to the controller. + let handle = std::thread::spawn(move || { + controller.serve_n_commands(1); + controller + }); + worker.shutdown("test done"); + let controller = handle.join().expect("mock joins"); + assert_eq!(controller.state(), crate::mock::MockState::SafeIdle); + } +} From 3f0d57f98e40ce4173abff2e6f57fb2a647869be Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:08:42 +0200 Subject: [PATCH 02/30] =?UTF-8?q?feat(stage-a-monitor):=20=E2=9C=A8=20add?= =?UTF-8?q?=20commissioning=20monitor=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live decimated photodiode waveform, calibrated clipping-guarded optical contrast, stream-integrity status, and gated manual controller actions (connect/config/start/stop/expert drive). Fails closed on the ABI v5 execution context: serial I/O only in the active live-capture worker; commands are host actions, never persistent settings. --- Cargo.toml | 1 + plugins/stage-a-monitor/Cargo.toml | 15 + plugins/stage-a-monitor/README.md | 39 ++ plugins/stage-a-monitor/plugin.toml | 7 + plugins/stage-a-monitor/src/lib.rs | 808 ++++++++++++++++++++++++++++ stage-a-io/src/transport.rs | 14 + 6 files changed, 884 insertions(+) create mode 100644 plugins/stage-a-monitor/Cargo.toml create mode 100644 plugins/stage-a-monitor/README.md create mode 100644 plugins/stage-a-monitor/plugin.toml create mode 100644 plugins/stage-a-monitor/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 98ff631..ca2a368 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "stage-a-io", + "plugins/stage-a-monitor", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/plugins/stage-a-monitor/Cargo.toml b/plugins/stage-a-monitor/Cargo.toml new file mode 100644 index 0000000..61dfffd --- /dev/null +++ b/plugins/stage-a-monitor/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "augur-plugin-stage-a-monitor" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A commissioning monitor: live photodiode readout, calibrated optical contrast, and gated manual Teensy drive control." + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde_json.workspace = true +stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-monitor/README.md b/plugins/stage-a-monitor/README.md new file mode 100644 index 0000000..8b43896 --- /dev/null +++ b/plugins/stage-a-monitor/README.md @@ -0,0 +1,39 @@ +# Stage-A Monitor + +Commissioning companion for the Stage-A camera-calibration bench: a live +view of the Teensy photodiode DAQ plus **gated** manual controller commands. + +## What it shows + +- **Photodiode waveform** — decimated calibrated trace (volts vs ms) from + the `SamplesU16` stream. +- **Live optical contrast** — `a = ln(V_max/V_min)` from dark-corrected, + clipping-guarded percentile extrema (see `stage-a-io::estimator`). An + invalid window shows *why* (clipped / no headroom / too short) instead of + a silently wrong number. +- **Stream integrity** — CRC failures, resync skips, frame-sequence gaps, + and ADC overruns. Any nonzero counter means the current point is invalid. + +## Controls (host actions on the status table) + +`Connect`, `Disconnect`, `Start acquisition`, `Stop`, and an expert +`Apply drive` modal (integer DAC codes; the optical contrast is always +measured, never assumed from the drive). Commands are actions — not +settings — so a reloaded settings file can never arm hardware. + +## Safety + +The plugin fails closed: the serial port opens only when the host reports +`LiveCapture` with `effects_allowed` (plugin ABI v5 execution context). +Replay and offline analysis can never emit a serial byte, and an existing +connection is shut down the moment effects are revoked. The firmware-side +watchdog independently drops the controller to `SAFE_IDLE` if the host +disappears. + +## Use it for (commissioning checklist) + +1. Wiring / voltage-range check at both detector loads. +2. Dark-level measurement for the estimator calibration. +3. Coherent-crosstalk test (H14): drive on, light blocked — the waveform + view and `a` readout must stay at the noise floor. +4. USB-throughput sanity (watch the integrity counters at full rate). diff --git a/plugins/stage-a-monitor/plugin.toml b/plugins/stage-a-monitor/plugin.toml new file mode 100644 index 0000000..b82d0f5 --- /dev/null +++ b/plugins/stage-a-monitor/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Monitor" +version = "0.2.0" +description = "Live Teensy photodiode readout, calibrated optical contrast, and gated manual drive control for Stage-A commissioning." +domain = "stage-a" +library = "augur_plugin_stage_a_monitor" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs new file mode 100644 index 0000000..f01c1f7 --- /dev/null +++ b/plugins/stage-a-monitor/src/lib.rs @@ -0,0 +1,808 @@ +//! Stage-A commissioning monitor. +//! +//! Live view of the Teensy photodiode DAQ (decimated waveform, calibrated +//! optical log-contrast `a`, clipping/headroom and stream-integrity status) +//! plus **gated** manual controller commands (connect, configure, start, +//! stop) for wiring and crosstalk commissioning. +//! +//! Safety contract (Stage-A control-software spec): +//! - devices open only when `HostContext::execution()` reports +//! `LiveCapture` **and** `effects_allowed` — replay and offline analysis +//! can never touch the serial port, and a stale worker is shut down the +//! moment the context stops permitting effects; +//! - commands are host actions, never persistent settings, so a reloaded +//! settings file cannot re-arm hardware; +//! - `process_frame()` only drains the bounded I/O worker queues. + +use std::collections::BTreeMap; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, + IoWorker, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, +}; + +const WAVEFORM_DATASET_ID: &str = "stage-a-monitor.waveform"; +const STATUS_DATASET_ID: &str = "stage-a-monitor.status"; +const WAVEFORM_VIEW_ID: &str = "stage-a-monitor.waveform.view"; +const STATUS_VIEW_ID: &str = "stage-a-monitor.status.view"; + +const ACTION_CONNECT: &str = "stage-a-monitor.connect"; +const ACTION_DISCONNECT: &str = "stage-a-monitor.disconnect"; +const ACTION_START: &str = "stage-a-monitor.start"; +const ACTION_STOP: &str = "stage-a-monitor.stop"; +const ACTION_APPLY_DRIVE: &str = "stage-a-monitor.apply-drive"; + +/// Retained sample window for the live view + contrast estimate. +const SAMPLE_RING_CAPACITY: usize = 32_768; +/// Points published per waveform refresh (decimated). +const WAVEFORM_POINTS: usize = 1_024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ConnectionState { + Disconnected, + Connected, + Acquiring, +} + +pub struct StageAMonitorPlugin { + enabled: bool, + // -- device -- + worker: Option, + connection: ConnectionState, + firmware: String, + next_tag: u64, + /// Tags of in-flight requests -> human-readable purpose. + in_flight: BTreeMap, + last_error: Option, + integrity: StreamIntegrity, + effects_blocked_reason: Option, + // -- settings -- + port_hint: String, + sample_rate_hz: i64, + block_samples: i64, + calibration: AdcCalibration, + // -- data -- + sample_ring: Vec, + ring_next_sample_index: u64, + sample_rate_seen_hz: u32, + contrast: Option, + contrast_error: Option, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAMonitorPlugin { + fn default() -> Self { + Self { + enabled: false, + worker: None, + connection: ConnectionState::Disconnected, + firmware: String::new(), + next_tag: 1, + in_flight: BTreeMap::new(), + last_error: None, + integrity: StreamIntegrity::default(), + effects_blocked_reason: None, + port_hint: "auto".into(), + sample_rate_hz: 20_000, + block_samples: 256, + calibration: AdcCalibration::default(), + sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), + ring_next_sample_index: 0, + sample_rate_seen_hz: 0, + contrast: None, + contrast_error: None, + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAMonitorPlugin { + fn bump_generation(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn connect(&mut self) { + if self.worker.is_some() { + return; + } + match open_transport(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + } + Err(err) => { + self.last_error = Some(err); + } + } + self.bump_generation(); + } + + fn disconnect(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + worker.shutdown(reason); + } + self.connection = ConnectionState::Disconnected; + self.in_flight.clear(); + self.bump_generation(); + } + + fn start_acquisition(&mut self) { + self.queue_command( + "config", + Command::new("CONFIG") + .field("mode", "A1") + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", self.block_samples) + .field("raw", 1) + .field("summary", 1), + ); + self.queue_command("start", Command::new("START")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(true)); + } + } + + fn stop_acquisition(&mut self) { + self.queue_command("stop", Command::new("STOP").field("reason", "operator")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + + fn apply_drive(&mut self, params: &Value) { + let freq_mhz = params.get("freq_mhz").and_then(Value::as_i64).unwrap_or(0); + let center_dac = params + .get("center_dac") + .and_then(Value::as_i64) + .unwrap_or(2_048); + let amplitude_dac = params + .get("amplitude_dac") + .and_then(Value::as_i64) + .unwrap_or(0); + self.queue_command( + "drive", + Command::new("CONFIG") + .field("mode", "A1") + .field("wave", "SINE") + .field("freq_mhz", freq_mhz) + .field("center_dac", center_dac) + .field("amplitude_dac", amplitude_dac) + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", self.block_samples) + .field("raw", 1) + .field("summary", 1), + ); + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + if outputs.is_empty() { + return; + } + let mut changed = false; + let mut stopped: Option = None; + for output in outputs { + changed = true; + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) => { + self.last_error = Some(format!("{purpose}: {err}")); + } + } + } + WorkerOutput::Event(DeviceEvent::Data(frame)) => { + match frame.header.frame_type { + FrameType::SamplesU16 => { + if let Some(codes) = frame.samples() { + self.sample_rate_seen_hz = frame.header.sample_rate_hz; + self.push_samples(&codes, frame.header.first_sample_index); + } + } + FrameType::Summary | FrameType::Marker | FrameType::Control => {} + FrameType::Unknown(_) => {} + } + } + WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Integrity(integrity) => { + self.integrity = integrity; + } + WorkerOutput::Stopped { reason } => { + stopped = Some(reason); + } + } + } + if let Some(reason) = stopped { + self.worker = None; + self.connection = ConnectionState::Disconnected; + self.last_error = Some(format!("device connection ended: {reason}")); + } + if changed { + self.refresh_contrast(); + self.bump_generation(); + } + } + + fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { + match purpose { + "hello" => { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + self.connection = ConnectionState::Connected; + } + "start" => { + self.connection = ConnectionState::Acquiring; + } + "stop" => { + self.connection = ConnectionState::Connected; + } + _ => {} + } + } + + fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { + self.ring_next_sample_index = first_sample_index + codes.len() as u64; + self.sample_ring.extend_from_slice(codes); + let len = self.sample_ring.len(); + if len > SAMPLE_RING_CAPACITY { + self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); + } + } + + fn refresh_contrast(&mut self) { + if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { + return; + } + match estimate_contrast(&self.sample_ring, &self.calibration) { + Ok(estimate) => { + self.contrast = Some(estimate); + self.contrast_error = None; + } + Err(err) => { + self.contrast = None; + self.contrast_error = Some(err.to_string()); + } + } + } + + fn waveform_dataset(&self) -> Series1dV1 { + let rate = if self.sample_rate_seen_hz > 0 { + f64::from(self.sample_rate_seen_hz) + } else { + self.sample_rate_hz as f64 + }; + let n = self.sample_ring.len(); + let stride = (n / WAVEFORM_POINTS).max(1); + let first_index = self.ring_next_sample_index.saturating_sub(n as u64); + let points: Vec = self + .sample_ring + .iter() + .enumerate() + .step_by(stride) + .map(|(i, &code)| Series1dPoint { + x: (first_index + i as u64) as f64 / rate * 1_000.0, + y: self.calibration.code_to_volts(code), + }) + .collect(); + Series1dV1 { + x_label: "time [ms]".into(), + y_label: "photodiode [V]".into(), + lines: vec![Series1dLine { + name: "photodiode".into(), + points, + }], + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.connection) { + (Some(reason), _) => format!("locked ({reason})"), + (None, ConnectionState::Disconnected) => "disconnected".into(), + (None, ConnectionState::Connected) => "connected".into(), + (None, ConnectionState::Acquiring) => "acquiring".into(), + }; + let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { + (Some(estimate), _) => ( + format!("{:.4}", estimate.a), + format!( + "{:.2}% low / {:.2}% high", + estimate.low_clip_fraction * 100.0, + estimate.high_clip_fraction * 100.0 + ), + ), + (None, Some(err)) => ("invalid".into(), err.clone()), + (None, None) => ("—".into(), "—".into()), + }; + let integrity = if self.integrity.is_clean() { + "clean".to_owned() + } else { + format!( + "crc={} gaps={} skipped={} overruns={}", + self.integrity.crc_failures, + self.integrity.sequence_gaps, + self.integrity.skipped_bytes, + self.integrity.dropped_samples + ) + }; + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("firmware", self.firmware.clone()), + text_column("a", a_text), + text_column("clipping", clip_text), + text_column("integrity", integrity), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("firmware", "Firmware"), + column("a", "a = ln(Vmax/Vmin)"), + column("clipping", "Clipping"), + column("integrity", "Stream integrity"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec<(String, Value)> { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-monitor.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push((request.action_id, request.params)); + } + consumed + } +} + +fn open_transport(port_hint: &str) -> Result, String> { + let path = resolve_port(port_hint)?; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +fn resolve_port(port_hint: &str) -> Result { + if port_hint != "auto" { + return Ok(port_hint.to_owned()); + } + let ports = serial_ports(); + ports + .into_iter() + .next() + .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned()) +} + +fn serial_ports() -> Vec { + serialport_names() + .into_iter() + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() +} + +fn serialport_names() -> Vec { + stage_a_io::transport::available_port_names() +} + +impl Plugin for StageAMonitorPlugin { + fn name(&self) -> &'static str { + "Stage-A Monitor" + } + + fn description(&self) -> &'static str { + "Live Teensy photodiode readout with calibrated optical contrast and gated manual drive control (commissioning)." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect("plugin disabled"); + } + } + + fn reset(&mut self) { + self.sample_ring.clear(); + self.contrast = None; + self.contrast_error = None; + self.bump_generation(); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Fail closed: any pass without live-capture effects tears the + // connection down and refuses commands. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.worker.is_some() { + self.disconnect("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for (action_id, params) in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect("operator"), + ACTION_START => self.start_acquisition(), + ACTION_STOP => self.stop_acquisition(), + ACTION_APPLY_DRIVE => self.apply_drive(¶ms), + _ => {} + } + } + + self.drain_worker(); + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Device".into(), + description: Some( + "Serial DAQ configuration. Connect/start/stop are actions on the status \ + table, never settings — a reloaded settings file can't arm hardware." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Serial port".into(), + tooltip: Some("Teensy USB serial device (auto = first usbmodem)".into()), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "sample_rate_hz".into(), + label: "ADC sample rate".into(), + tooltip: Some("Commanded photodiode sample rate".into()), + kind: SettingKind::I64Slider { + min: 1_000, + max: 100_000, + default: self.sample_rate_hz, + suffix: Some(" Hz".into()), + }, + }, + SettingItem { + key: "dark_millivolts".into(), + label: "Dark level".into(), + tooltip: Some( + "Light-blocked photodiode level; a is computed from dark-corrected \ + voltages" + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 3_300.0, + speed: 1.0, + default: self.calibration.dark_volts * 1_000.0, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "sample_rate_hz" => Some(json!(self.sample_rate_hz)), + "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "sample_rate_hz" => { + self.sample_rate_hz = value + .as_i64() + .ok_or("sample_rate_hz must be an integer")? + .clamp(1_000, 100_000); + Ok(()) + } + "dark_millivolts" => { + let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; + self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + entries.push(StatusEntry::Text(match self.connection { + ConnectionState::Disconnected => "Teensy: disconnected".into(), + ConnectionState::Connected => format!("Teensy: connected ({})", self.firmware), + ConnectionState::Acquiring => format!( + "Teensy: acquiring at {} S/s", + if self.sample_rate_seen_hz > 0 { + self.sample_rate_seen_hz as i64 + } else { + self.sample_rate_hz + } + ), + })); + if let Some(estimate) = &self.contrast { + entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: WAVEFORM_DATASET_ID.into(), + title: "Photodiode waveform".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No photodiode samples yet — connect and start.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Stage-A monitor status".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Monitor idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: WAVEFORM_VIEW_ID.into(), + title: "Photodiode".into(), + dataset_id: WAVEFORM_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Monitor status".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + HostActionDescriptor { + id: ACTION_CONNECT.into(), + title: "Connect".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_DISCONNECT.into(), + title: "Disconnect".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_START.into(), + title: "Start acquisition".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_STOP.into(), + title: "Stop".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_APPLY_DRIVE.into(), + title: "Apply drive (expert)".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: serde_json::to_value(SettingsSchema { + sections: vec![SettingsSection { + label: "Drive".into(), + description: Some( + "Integer DAC drive codes — the optical contrast is measured \ + from the photodiode, never assumed from these values." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "freq_mhz".into(), + label: "Frequency".into(), + tooltip: Some("Drive frequency in millihertz".into()), + kind: SettingKind::I64Drag { + min: 0, + max: 200_000_000, + default: 1_000_000, + }, + }, + SettingItem { + key: "center_dac".into(), + label: "Center DAC code".into(), + tooltip: None, + kind: SettingKind::I64Drag { + min: 0, + max: 4_095, + default: 2_048, + }, + }, + SettingItem { + key: "amplitude_dac".into(), + label: "Amplitude DAC code".into(), + tooltip: None, + kind: SettingKind::I64Drag { + min: 0, + max: 2_047, + default: 0, + }, + }, + ], + }], + }) + .ok(), + }, + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), + _ => 0, + } + } +} + +impl Drop for StageAMonitorPlugin { + fn drop(&mut self) { + self.disconnect("plugin destroyed"); + } +} + +export_plugin!(StageAMonitorPlugin); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn waveform_dataset_decimates_and_calibrates() { + let mut plugin = StageAMonitorPlugin::default(); + plugin.sample_rate_seen_hz = 20_000; + plugin.push_samples(&vec![2_048_u16; 8_192], 0); + let dataset = plugin.waveform_dataset(); + assert_eq!(dataset.lines.len(), 1); + assert!(dataset.lines[0].points.len() <= WAVEFORM_POINTS + 1); + let volts = dataset.lines[0].points[0].y; + assert!((volts - 2_048.0 * 3.3 / 4_095.0).abs() < 1e-9); + } + + #[test] + fn sample_ring_is_bounded() { + let mut plugin = StageAMonitorPlugin::default(); + plugin.push_samples(&vec![1_u16; SAMPLE_RING_CAPACITY], 0); + plugin.push_samples(&vec![2_u16; 4_096], SAMPLE_RING_CAPACITY as u64); + assert_eq!(plugin.sample_ring.len(), SAMPLE_RING_CAPACITY); + assert_eq!(*plugin.sample_ring.last().unwrap(), 2); + } + + #[test] + fn status_dataset_matches_its_schema() { + let plugin = StageAMonitorPlugin::default(); + let dataset = plugin.status_dataset(); + let schema = plugin.status_schema(); + assert_eq!(dataset.columns.len(), schema.columns.len()); + for (data, column) in dataset.columns.iter().zip(&schema.columns) { + assert_eq!(data.column_id, column.id); + assert_eq!(data.len(), 1); + } + } +} diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index 27dea22..fc8482a 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -111,3 +111,17 @@ impl Transport for MockTransport { Ok(()) } } + +/// Names of serial ports visible to the OS (empty without the `hardware` +/// feature). Used by plugins to offer a port picker. +#[cfg(feature = "hardware")] +pub fn available_port_names() -> Vec { + serialport::available_ports() + .map(|ports| ports.into_iter().map(|p| p.port_name).collect()) + .unwrap_or_default() +} + +#[cfg(not(feature = "hardware"))] +pub fn available_port_names() -> Vec { + Vec::new() +} From 8f448c2c53a5fd277fe3177754db81df34dc2349 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:17:09 +0200 Subject: [PATCH 03/30] =?UTF-8?q?feat(stage-a-a1):=20=E2=9C=A8=20add=20A1?= =?UTF-8?q?=20minimum-depth=20Bode=20calibration=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Statistical core: phase folding (hardware EXT_TRIGGER fiducials or software clock-skew recovery via Rayleigh-power frequency scan), Rayleigh detection with Bonferroni-charged trials, median-background phase-locked excess, probit a_min fit with profile CI, hot-pixel mask from an unmodulated reference window. Sweep engine bisects the drive code, grids the bracketed transition, and records measured optical contrast per point. PDQ + sidecar + results export per run; fails closed on the ABI v5 execution context. --- Cargo.toml | 1 + plugins/stage-a-a1/Cargo.toml | 15 + plugins/stage-a-a1/README.md | 66 ++ plugins/stage-a-a1/plugin.toml | 7 + plugins/stage-a-a1/src/analysis.rs | 603 ++++++++++++++ plugins/stage-a-a1/src/lib.rs | 1224 ++++++++++++++++++++++++++++ plugins/stage-a-a1/src/sweep.rs | 363 +++++++++ 7 files changed, 2279 insertions(+) create mode 100644 plugins/stage-a-a1/Cargo.toml create mode 100644 plugins/stage-a-a1/README.md create mode 100644 plugins/stage-a-a1/plugin.toml create mode 100644 plugins/stage-a-a1/src/analysis.rs create mode 100644 plugins/stage-a-a1/src/lib.rs create mode 100644 plugins/stage-a-a1/src/sweep.rs diff --git a/Cargo.toml b/Cargo.toml index ca2a368..e2b3d5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "stage-a-io", "plugins/stage-a-monitor", + "plugins/stage-a-a1", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/plugins/stage-a-a1/Cargo.toml b/plugins/stage-a-a1/Cargo.toml new file mode 100644 index 0000000..fa4841d --- /dev/null +++ b/plugins/stage-a-a1/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "augur-plugin-stage-a-a1" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A A1 event-native Bode calibration: minimum-depth a_min(f) sweep with phase-locked detection." + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde_json.workspace = true +stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md new file mode 100644 index 0000000..48055cf --- /dev/null +++ b/plugins/stage-a-a1/README.md @@ -0,0 +1,66 @@ +# Stage-A A1 — minimum-depth Bode calibration + +Measures `a_min(f)`: the smallest optical log-contrast that still produces +phase-locked camera events, per drive frequency. `|H(f)| = C / a_min(f)`; +the knee of the curve is the pixel bandwidth `f_c(I)`, and the plateau of +`a_min` reads out the contrast quantum `C` (which seeds A3). Protocol +design: knowledge base `methodology/camera-calibration.md` (A1) and +`methodology/stage-a-control-software.md`. + +## How it decides "events just appeared" + +- **Detector — phase, not counts.** Background activity is uniform in + drive phase; modulation events are phase-locked. Each measurement window + is folded and tested with the **Rayleigh test**; background is discounted + automatically instead of subtracting a drifting absolute rate. +- **Cycle fiducial.** With the phase-0 TTL wired into `EXT_TRIGGER`, the + camera-clock edges from `frame.external_triggers()` mark each cycle. + Without the cable, the drive frequency is **refined against the events** + (Rayleigh-power scan over ±ppm around the commanded value — recovers the + Teensy↔camera clock skew); the scan multiplicity is Bonferroni-charged + to the significance threshold. +- **Estimator.** The mean phase-locked events per half-cycle comes from the + positive excess over the median phase-bin occupancy. +- **a_min is a fitted crossing.** The 0→1 step is smeared by shot-noise + first-passage randomness and per-pixel threshold dispersion, so a_min is + the fitted `N = 0.5` crossing of a probit in `ln a`, with a profile + confidence interval. The fitted transition width is a free preview of + the smear (σ_C + FPT). +- **Hot pixels.** An unmodulated reference window at run start builds a + median+5·MAD mask; masked pixels never enter the statistics, and the + mask size is recorded in the sidecar. +- **`a` is measured light.** Every point's contrast comes from the + photodiode ADC through the calibrated, clipping-guarded estimator in + `stage-a-io` — never from the commanded DAC code. Invalid windows + (clipping, CRC/sequence/overrun faults) are re-measured, never patched. + +## Run flow + +`Arm controller` → `Run A1 sweep`: reference window (hot-pixel mask) → +per frequency: bisection on the drive code until the detection boundary is +bracketed → log-spaced grid across the transition → probit fit → +next frequency. Views: `a_min(f)` with CI, live phase histogram, `N(a)` +staircase, run status. Raw PDA1 frames go to +`~/.augur/stage-a-runs/.pdq` with a JSON sidecar and a results +export; final numbers must be recomputed from the camera RAW + PDQ. + +ON and OFF are measured in **separate runs** (settings → Polarity) — the +comparator paths are asymmetric and must never be pooled. + +## Safety + +Fails closed on the ABI v5 execution context exactly like +`stage-a-monitor`: serial I/O only in the active live-capture worker; +Arm/Run/Stop are host actions, never settings; the firmware watchdog +drops to `SAFE_IDLE` independently of host cleanup. + +## Current limitations + +- The Teensy DDS/DAC firmware is still the ADC-only commissioning build — + closed-loop sweeps run against the protocol but the final stimulus + backend is blocked on the hardware freeze (see `stage-a-controller`). +- Marker cycles (periodic full-depth optical anchors) are specced for the + firmware but not yet emitted; the software frequency lock covers the + missing-trigger-cable case meanwhile. +- Measured sample cadence validation and the A5 refractory validity bound + `2fa/C ≪ 1/τ_refr` are recorded, not yet enforced. diff --git a/plugins/stage-a-a1/plugin.toml b/plugins/stage-a-a1/plugin.toml new file mode 100644 index 0000000..62f6089 --- /dev/null +++ b/plugins/stage-a-a1/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A A1 Min-Depth" +version = "0.2.0" +description = "Event-native Bode calibration: a_min(f) via phase-locked detection, drive bisection, and probit fitting." +domain = "stage-a" +library = "augur_plugin_stage_a_a1" +phase = "raw_events" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-a1/src/analysis.rs b/plugins/stage-a-a1/src/analysis.rs new file mode 100644 index 0000000..32cce7c --- /dev/null +++ b/plugins/stage-a-a1/src/analysis.rs @@ -0,0 +1,603 @@ +//! Statistical core of the A1 minimum-depth measurement. +//! +//! ## Why phase, not raw counts +//! +//! Background activity (BA) is uniform in modulation phase; genuine +//! modulation events are phase-locked to the drive. Testing for a +//! phase-locked component (Rayleigh test) therefore discounts uniform +//! background *automatically*, instead of requiring an absolute background +//! rate that drifts with temperature. "Mean events per half-cycle > 1" is +//! kept as the *estimator* (it is the quantity `⌊a·|H|/C⌋` predicts), but +//! the *detector* is the phase test. +//! +//! ## Cycle fiducials without the trigger cable +//! +//! With the phase-0 TTL wired, `frame.external_triggers()` marks each cycle +//! on the camera clock. Without it, the Teensy and camera clocks drift +//! (tens of ppm — folding dies after ~0.1 s at 10 kHz), so the drive +//! frequency is *refined against the events themselves*: scan a small +//! window around the commanded frequency and keep the value maximising the +//! Rayleigh power. The scan multiplicity is charged to the significance +//! test (Bonferroni). +//! +//! ## a_min as a fitted crossing +//! +//! Near threshold the 0→1 step of `⌊a·|H|/C⌋` is smeared by shot-noise +//! first-passage randomness and per-pixel threshold dispersion, so "events +//! just vanish" is not a crisp edge. a_min is defined as the fitted point +//! where the mean phase-locked events per half-cycle crosses 0.5, from a +//! probit-in-ln(a) fit over the transition, with a profile confidence +//! interval. The plateau of a_min(f) reads out the contrast quantum C. + +// --------------------------------------------------------------------------- +// Phase folding +// --------------------------------------------------------------------------- + +/// Folds event timestamps at `frequency_hz` relative to `t0_us`, +/// returning phases in `[0, 1)`. +pub fn fold_phases( + timestamps_us: impl Iterator, + t0_us: u64, + frequency_hz: f64, +) -> Vec { + let period_us = 1.0e6 / frequency_hz; + timestamps_us + .map(|t| { + let dt = t.saturating_sub(t0_us) as f64; + (dt / period_us).fract() + }) + .collect() +} + +/// Folds against explicit cycle-start fiducials (rising trigger edges): +/// each event's phase is its position inside the enclosing cycle. Events +/// before the first or after the last fiducial are dropped (their cycle +/// length is unknown). +pub fn fold_phases_with_fiducials(events: &[u64], cycle_starts_us: &[u64]) -> Vec { + if cycle_starts_us.len() < 2 { + return Vec::new(); + } + let mut phases = Vec::with_capacity(events.len()); + for &t in events { + let idx = match cycle_starts_us.binary_search(&t) { + Ok(i) => i, + Err(0) => continue, + Err(i) => i - 1, + }; + if idx + 1 >= cycle_starts_us.len() { + continue; + } + let start = cycle_starts_us[idx]; + let end = cycle_starts_us[idx + 1]; + if end <= start { + continue; + } + phases.push((t - start) as f64 / (end - start) as f64); + } + phases +} + +// --------------------------------------------------------------------------- +// Rayleigh test +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RayleighResult { + pub n: usize, + /// Resultant length in [0, 1]. + pub r: f64, + /// Z = n·R². + pub z: f64, + /// Approximate p-value under uniformity, `exp(-Z)` with the standard + /// small-sample correction (Zar / Wilkie). + pub p_value: f64, +} + +pub fn rayleigh_test(phases: &[f64]) -> RayleighResult { + let n = phases.len(); + if n == 0 { + return RayleighResult { + n, + r: 0.0, + z: 0.0, + p_value: 1.0, + }; + } + let (mut c, mut s) = (0.0_f64, 0.0_f64); + for &phase in phases { + let angle = 2.0 * std::f64::consts::PI * phase; + c += angle.cos(); + s += angle.sin(); + } + let r = (c * c + s * s).sqrt() / n as f64; + let z = n as f64 * r * r; + let nf = n as f64; + let p = (-z).exp() * (1.0 + (2.0 * z - z * z) / (4.0 * nf) + - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); + RayleighResult { + n, + r, + z, + p_value: p.clamp(0.0, 1.0), + } +} + +// --------------------------------------------------------------------------- +// Frequency refinement (clock-skew recovery without a trigger cable) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FrequencyLock { + pub frequency_hz: f64, + pub rayleigh: RayleighResult, + /// Number of candidate frequencies tested — multiply into the + /// significance threshold (Bonferroni). + pub trials: usize, +} + +/// Scans `±window_ppm` around `nominal_hz` and returns the frequency with +/// the maximum Rayleigh power. The step is chosen so consecutive candidates +/// dephase by ≤ 0.1 cycle over the observation span (finer is wasted). +pub fn refine_frequency( + timestamps_us: &[u64], + nominal_hz: f64, + window_ppm: f64, +) -> Option { + let (&first, &last) = (timestamps_us.first()?, timestamps_us.last()?); + let span_s = (last.saturating_sub(first)) as f64 / 1.0e6; + if span_s <= 0.0 { + return None; + } + let df_step = 0.1 / span_s; + let half_window_hz = nominal_hz * window_ppm * 1e-6; + let steps = ((half_window_hz / df_step).ceil() as i64).clamp(0, 5_000); + let mut best: Option = None; + let trials = (2 * steps + 1) as usize; + for k in -steps..=steps { + let f = nominal_hz + k as f64 * df_step; + if f <= 0.0 { + continue; + } + let phases = fold_phases(timestamps_us.iter().copied(), first, f); + let stat = rayleigh_test(&phases); + if best.as_ref().is_none_or(|b| stat.z > b.rayleigh.z) { + best = Some(FrequencyLock { + frequency_hz: f, + rayleigh: stat, + trials, + }); + } + } + best +} + +// --------------------------------------------------------------------------- +// Phase-locked excess (the events/half-cycle estimator) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseHistogram { + pub bins: Vec, + pub total: usize, +} + +pub fn phase_histogram(phases: &[f64], bin_count: usize) -> PhaseHistogram { + let mut bins = vec![0_u32; bin_count.max(1)]; + for &phase in phases { + let idx = ((phase * bins.len() as f64) as usize).min(bins.len() - 1); + bins[idx] += 1; + } + PhaseHistogram { + bins, + total: phases.len(), + } +} + +/// Estimates the phase-locked event count above the uniform background. +/// +/// The per-bin background is the *median* bin occupancy — robust because +/// the locked cluster occupies a minority of bins. Returns the summed +/// positive excess. Dividing by the number of observed cycles gives the +/// mean phase-locked events per cycle (per polarity: one burst per cycle). +pub fn phase_locked_excess(histogram: &PhaseHistogram) -> f64 { + if histogram.bins.is_empty() { + return 0.0; + } + let mut sorted = histogram.bins.clone(); + sorted.sort_unstable(); + let median = f64::from(sorted[sorted.len() / 2]); + histogram + .bins + .iter() + .map(|&count| (f64::from(count) - median).max(0.0)) + .sum() +} + +// --------------------------------------------------------------------------- +// Detection verdict for one (frequency, amplitude) measurement +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DetectionVerdict { + pub detected: bool, + pub p_value: f64, + /// Bonferroni-corrected significance threshold actually applied. + pub alpha_effective: f64, + /// Mean phase-locked events per cycle (per polarity), background-free. + pub locked_events_per_cycle: f64, +} + +/// Decides whether phase-locked modulation events are present. +/// +/// `alpha` is the per-measurement false-positive budget; `trials` is the +/// look-elsewhere multiplicity (frequency-scan candidates × bisection +/// steps), charged via Bonferroni. +pub fn detect( + rayleigh: RayleighResult, + excess: f64, + observed_cycles: f64, + alpha: f64, + trials: usize, +) -> DetectionVerdict { + let alpha_effective = alpha / trials.max(1) as f64; + DetectionVerdict { + detected: rayleigh.p_value < alpha_effective, + p_value: rayleigh.p_value, + alpha_effective, + locked_events_per_cycle: if observed_cycles > 0.0 { + excess / observed_cycles + } else { + 0.0 + }, + } +} + +// --------------------------------------------------------------------------- +// a_min fit: probit in ln(a) with profile CI +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MinDepthFit { + /// a at which the mean locked events/half-cycle crosses 0.5. + pub a_min: f64, + /// Profile interval (Δ SSE ≤ SSE_min · (1 + 2/dof)); honest-but-cheap. + pub a_min_low: f64, + pub a_min_high: f64, + /// Transition width in ln(a) — first look at σ_C + FPT smear. + pub sigma_ln_a: f64, + pub points_used: usize, +} + +/// One measured amplitude point for the fit. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DepthPoint { + /// Measured optical log-contrast (photodiode, never the drive code). + pub a: f64, + /// Mean phase-locked events per half-cycle at this contrast. + pub events_per_half_cycle: f64, +} + +fn standard_normal_cdf(z: f64) -> f64 { + // Abramowitz & Stegun 7.1.26 via erf; |error| < 1.5e-7. + let x = z / std::f64::consts::SQRT_2; + let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); + let poly = t + * (0.254_829_592 + + t * (-0.284_496_736 + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); + let erf_abs = 1.0 - poly * (-x * x).exp(); + let erf = if x >= 0.0 { erf_abs } else { -erf_abs }; + 0.5 * (1.0 + erf) +} + +/// Fits `N(a) = Φ((ln a − μ)/σ)` over the transition region and reports +/// `a_min = e^μ` (the N = 0.5 crossing). Points far above the first step +/// (`N > 1.5`) are excluded — there the staircase's higher steps dominate +/// and the single-step model no longer applies. +pub fn fit_min_depth(points: &[DepthPoint]) -> Option { + let usable: Vec = points + .iter() + .copied() + .filter(|p| p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5) + .collect(); + if usable.len() < 3 { + return None; + } + let has_low = usable.iter().any(|p| p.events_per_half_cycle < 0.4); + let has_high = usable.iter().any(|p| p.events_per_half_cycle > 0.6); + if !has_low || !has_high { + return None; + } + + let ln_min = usable.iter().map(|p| p.a.ln()).fold(f64::INFINITY, f64::min); + let ln_max = usable + .iter() + .map(|p| p.a.ln()) + .fold(f64::NEG_INFINITY, f64::max); + + let sse = |mu: f64, sigma: f64| -> f64 { + usable + .iter() + .map(|p| { + let model = standard_normal_cdf((p.a.ln() - mu) / sigma); + let d = p.events_per_half_cycle.min(1.0) - model; + d * d + }) + .sum() + }; + + let mut best = (f64::INFINITY, ln_min, 0.1); + let mu_steps = 200; + for i in 0..=mu_steps { + let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; + for j in 0..40 { + let sigma = 0.005 * 1.2_f64.powi(j); // 0.005 .. ~7 in ln a + let value = sse(mu, sigma); + if value < best.0 { + best = (value, mu, sigma); + } + } + } + let (sse_min, mu_hat, sigma_hat) = best; + let dof = usable.len().saturating_sub(2).max(1) as f64; + let threshold = sse_min * (1.0 + 2.0 / dof) + 1e-12; + + // Profile over mu: the interval where some sigma keeps SSE under the + // threshold. + let mut low = mu_hat; + let mut high = mu_hat; + for i in 0..=mu_steps { + let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; + let feasible = (0..40).any(|j| { + let sigma = 0.005 * 1.2_f64.powi(j); + sse(mu, sigma) <= threshold + }); + if feasible { + low = low.min(mu); + high = high.max(mu); + } + } + + Some(MinDepthFit { + a_min: mu_hat.exp(), + a_min_low: low.exp(), + a_min_high: high.exp(), + sigma_ln_a: sigma_hat, + points_used: usable.len(), + }) +} + +// --------------------------------------------------------------------------- +// Hot-pixel mask (background is heavy-tailed; mask the tail, use the body) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct HotPixelMask { + width: u16, + masked: Vec, +} + +impl HotPixelMask { + /// Builds the mask from per-pixel counts of an *unmodulated* reference + /// window: pixels above `median + 5·MAD` (and above a small absolute + /// floor) are masked. The mask is fixed-pattern and belongs in the run + /// metadata, not just preprocessing. + pub fn from_reference_counts(width: u16, _height: u16, counts: &[u32]) -> Self { + let mut sorted: Vec = counts.to_vec(); + sorted.sort_unstable(); + let median = sorted.get(sorted.len() / 2).copied().unwrap_or(0) as f64; + let mut deviations: Vec = counts + .iter() + .map(|&count| (f64::from(count) - median).abs()) + .collect(); + deviations.sort_by(f64::total_cmp); + let mad = deviations.get(deviations.len() / 2).copied().unwrap_or(0.0); + let threshold = median + 5.0 * mad.max(0.5) + 2.0; + let masked = counts + .iter() + .map(|&count| f64::from(count) > threshold) + .collect(); + Self { width, masked } + } + + pub fn is_masked(&self, x: u16, y: u16) -> bool { + self.masked + .get(y as usize * self.width as usize + x as usize) + .copied() + .unwrap_or(false) + } + + pub fn masked_count(&self) -> usize { + self.masked.iter().filter(|&&m| m).count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Deterministic pseudo-uniform stream (splitmix64 → [0,1)). + struct UniformStream { + state: u64, + } + + impl UniformStream { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next(&mut self) -> f64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z = z ^ (z >> 31); + (z >> 11) as f64 / (1_u64 << 53) as f64 + } + + fn take(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.next()).collect() + } + } + + fn uniform_sequence(seed: u64, n: usize) -> Vec { + UniformStream::new(seed).take(n) + } + + /// Synthetic event stream: `per_cycle` phase-locked events per cycle at + /// `locked_phase` (jitter ±0.02) plus `background_rate_hz` uniform noise. + fn synthetic_events( + frequency_hz: f64, + duration_s: f64, + per_cycle: f64, + background_rate_hz: f64, + seed: u64, + ) -> Vec { + let cycles = (frequency_hz * duration_s) as usize; + let period_us = 1.0e6 / frequency_hz; + let mut stream = UniformStream::new(seed); + let mut next = move || stream.next(); + let mut events = Vec::new(); + for cycle in 0..cycles { + let base = cycle as f64 * period_us; + // Bernoulli(per_cycle fractional part) + floor. + let mut count = per_cycle.floor() as usize; + if next() < per_cycle.fract() { + count += 1; + } + for _ in 0..count { + let phase = 0.25 + (next() - 0.5) * 0.04; + events.push((base + phase * period_us) as u64); + } + } + let n_background = (background_rate_hz * duration_s) as usize; + for _ in 0..n_background { + events.push((next() * duration_s * 1.0e6) as u64); + } + events.sort_unstable(); + events + } + + #[test] + fn rayleigh_accepts_uniform_and_rejects_locked_phases() { + let uniform = uniform_sequence(7, 2_000); + let stat = rayleigh_test(&uniform); + assert!(stat.p_value > 0.01, "uniform phases: p={}", stat.p_value); + + let locked: Vec = uniform_sequence(11, 200) + .into_iter() + .map(|u| 0.3 + 0.02 * (u - 0.5)) + .collect(); + let stat = rayleigh_test(&locked); + assert!(stat.p_value < 1e-12, "locked phases: p={}", stat.p_value); + } + + #[test] + fn detection_discounts_uniform_background() { + // 0.8 locked events/cycle at 1 kHz for 0.5 s, drowned in 10x + // background rate: still detected via phase. + let events = synthetic_events(1_000.0, 0.5, 0.8, 8_000.0, 3); + let phases = fold_phases(events.iter().copied(), 0, 1_000.0); + let stat = rayleigh_test(&phases); + assert!(stat.p_value < 1e-6, "p={}", stat.p_value); + + // Background alone must NOT detect. + let noise_only = synthetic_events(1_000.0, 0.5, 0.0, 8_000.0, 5); + let phases = fold_phases(noise_only.iter().copied(), 0, 1_000.0); + let stat = rayleigh_test(&phases); + assert!(stat.p_value > 1e-3, "background-only p={}", stat.p_value); + } + + #[test] + fn phase_locked_excess_recovers_events_per_cycle() { + let frequency = 2_000.0; + let duration = 0.5; + let per_cycle = 0.6; + let events = synthetic_events(frequency, duration, per_cycle, 2_000.0, 9); + let phases = fold_phases(events.iter().copied(), 0, frequency); + let histogram = phase_histogram(&phases, 32); + let cycles = frequency * duration; + let recovered = phase_locked_excess(&histogram) / cycles; + assert!( + (recovered - per_cycle).abs() < 0.12, + "recovered {recovered} vs {per_cycle}" + ); + } + + #[test] + fn frequency_refinement_recovers_clock_skew() { + // Commanded 5 kHz, true (camera-clock) frequency 300 ppm higher — + // the naive fold dephases by 1.5 cycles over the 1 s span and + // collapses, while the refined lock recovers the true frequency. + let true_hz = 5_000.0 * (1.0 + 300e-6); + let events = synthetic_events(true_hz, 1.0, 1.0, 500.0, 13); + let lock = refine_frequency(&events, 5_000.0, 500.0).expect("lock found"); + let recovered_ppm = (lock.frequency_hz / 5_000.0 - 1.0) * 1e6; + // The scan step is 0.1/span = 0.1 Hz = 20 ppm at 5 kHz. + assert!( + (recovered_ppm - 300.0).abs() < 25.0, + "recovered {recovered_ppm} ppm" + ); + let naive = rayleigh_test(&fold_phases(events.iter().copied(), events[0], 5_000.0)); + assert!( + lock.rayleigh.z > naive.z * 5.0, + "lock z={} naive z={}", + lock.rayleigh.z, + naive.z + ); + } + + #[test] + fn fiducial_folding_matches_known_phase() { + let cycle_starts: Vec = (0..100).map(|k| k * 1_000).collect(); + let events: Vec = (0..99).map(|k| k * 1_000 + 250).collect(); + let phases = fold_phases_with_fiducials(&events, &cycle_starts); + assert_eq!(phases.len(), 99); + assert!(phases.iter().all(|p| (p - 0.25).abs() < 1e-9)); + } + + #[test] + fn min_depth_fit_recovers_the_crossing() { + // True a_min = 0.20, smear sigma = 0.15 in ln a. + let mu = 0.2_f64.ln(); + let points: Vec = (0..12) + .map(|i| { + let a = 0.08 * 1.25_f64.powi(i); // 0.08 .. ~0.9 + DepthPoint { + a, + events_per_half_cycle: standard_normal_cdf((a.ln() - mu) / 0.15), + } + }) + .collect(); + let fit = fit_min_depth(&points).expect("fit succeeds"); + assert!( + (fit.a_min - 0.2).abs() < 0.02, + "a_min={} (expected 0.20)", + fit.a_min + ); + assert!(fit.a_min_low <= fit.a_min && fit.a_min <= fit.a_min_high); + assert!((fit.sigma_ln_a - 0.15).abs() < 0.08); + } + + #[test] + fn min_depth_fit_requires_a_bracketed_transition() { + // All points fully above threshold: no crossing to fit. + let points: Vec = (0..6) + .map(|i| DepthPoint { + a: 0.5 + 0.1 * i as f64, + events_per_half_cycle: 1.0, + }) + .collect(); + assert!(fit_min_depth(&points).is_none()); + } + + #[test] + fn hot_pixel_mask_flags_the_tail_only() { + let mut counts = vec![2_u32; 64 * 64]; + counts[5] = 500; // hot + counts[700] = 300; // hot + let mask = HotPixelMask::from_reference_counts(64, 64, &counts); + assert_eq!(mask.masked_count(), 2); + assert!(mask.is_masked(5, 0)); + assert!(!mask.is_masked(6, 0)); + } +} diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs new file mode 100644 index 0000000..c1fd032 --- /dev/null +++ b/plugins/stage-a-a1/src/lib.rs @@ -0,0 +1,1224 @@ +//! Stage-A A1 — event-native Bode calibration, minimum-depth method. +//! +//! Measures `a_min(f)`: the smallest optical log-contrast that still +//! produces phase-locked events, per drive frequency. `|H(f)| = +//! C/a_min(f)`, the knee is `f_c(I)`, and the plateau of `a_min` reads out +//! the contrast quantum `C` (knowledge base: +//! `methodology/camera-calibration.md`, A1 protocol). +//! +//! Division of labour: +//! - `analysis` — phase folding, Rayleigh detection, frequency-skew +//! recovery, phase-locked excess, probit `a_min` fit, hot-pixel mask; +//! - `sweep` — the per-frequency bisection/grid state machine; +//! - this module — device I/O through `stage-a-io` (gated by the ABI v5 +//! execution context), camera-event intake, measurement windows, live +//! views, and the run sidecar. +//! +//! ON and OFF are measured **separately** (never pooled — the paths are +//! asymmetric); select the polarity in the settings and run each sweep. + +mod analysis; +mod sweep; + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, PluginInput, + Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, Command, DeviceEvent, FrameType, IoWorker, PdqWriter, + RunSidecar, StageAClient, StreamIntegrity, TriggerSource, WorkerOutput, WorkerRequest, +}; + +use analysis::{ + detect, fold_phases, fold_phases_with_fiducials, phase_histogram, phase_locked_excess, + rayleigh_test, refine_frequency, HotPixelMask, PhaseHistogram, +}; +use sweep::{Measurement, SweepCommand, SweepEngine, SweepPlan}; + +const AMIN_DATASET_ID: &str = "stage-a-a1.amin"; +const PHASE_DATASET_ID: &str = "stage-a-a1.phase"; +const DEPTH_DATASET_ID: &str = "stage-a-a1.depth"; +const STATUS_DATASET_ID: &str = "stage-a-a1.status"; + +const ACTION_ARM: &str = "stage-a-a1.arm"; +const ACTION_RUN: &str = "stage-a-a1.run"; +const ACTION_STOP: &str = "stage-a-a1.stop"; + +const PHASE_BINS: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RunState { + Idle, + Armed, + Reference, + Sweeping, + Finished, +} + +/// Camera-side accumulation for the current measurement window. +#[derive(Default)] +struct WindowAccumulator { + /// Camera timestamps of polarity-selected, hot-pixel-filtered events. + event_timestamps_us: Vec, + /// Rising phase-0 trigger edges (cycle fiducials) inside the window. + trigger_edges_us: Vec, + /// ADC codes streamed by the Teensy during the window. + adc_codes: Vec, + window_start_us: Option, + latest_camera_ts_us: u64, +} + +impl WindowAccumulator { + fn clear(&mut self) { + self.event_timestamps_us.clear(); + self.trigger_edges_us.clear(); + self.adc_codes.clear(); + self.window_start_us = None; + } + + fn elapsed_us(&self) -> u64 { + self.window_start_us + .map(|start| self.latest_camera_ts_us.saturating_sub(start)) + .unwrap_or(0) + } +} + +pub struct StageAA1Plugin { + enabled: bool, + state: RunState, + // device + worker: Option, + next_tag: u64, + in_flight: BTreeMap, + firmware: String, + integrity: StreamIntegrity, + last_error: Option, + effects_blocked_reason: Option, + // configuration (settings) + port_hint: String, + polarity_on: bool, + freq_start_hz: f64, + freq_stop_hz: f64, + points_per_decade: i64, + cycles_per_measurement: i64, + settle_ms: i64, + alpha: f64, + initial_amplitude_dac: i64, + sample_rate_hz: i64, + calibration: AdcCalibration, + // run + engine: Option, + window: WindowAccumulator, + settle_until_us: Option, + hot_pixels: Option, + reference_counts: Vec, + sensor_size: (u16, u16), + run_id: String, + pdq: Option, + current_phase_histogram: Option, + used_hardware_fiducial: bool, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAA1Plugin { + fn default() -> Self { + Self { + enabled: false, + state: RunState::Idle, + worker: None, + next_tag: 1, + in_flight: BTreeMap::new(), + firmware: String::new(), + integrity: StreamIntegrity::default(), + last_error: None, + effects_blocked_reason: None, + port_hint: "auto".into(), + polarity_on: true, + freq_start_hz: 100.0, + freq_stop_hz: 50_000.0, + points_per_decade: 6, + cycles_per_measurement: 400, + settle_ms: 100, + alpha: 0.001, + initial_amplitude_dac: 512, + sample_rate_hz: 20_000, + calibration: AdcCalibration::default(), + engine: None, + window: WindowAccumulator::default(), + settle_until_us: None, + hot_pixels: None, + reference_counts: Vec::new(), + sensor_size: (0, 0), + run_id: String::new(), + pdq: None, + current_phase_histogram: None, + used_hardware_fiducial: false, + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAA1Plugin { + fn bump(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn frequency_grid(&self) -> Vec { + let start = self.freq_start_hz.max(1.0); + let stop = self.freq_stop_hz.max(start * 1.01); + let per_decade = self.points_per_decade.max(1) as f64; + let decades = (stop / start).log10(); + let n = (decades * per_decade).ceil() as usize + 1; + (0..n) + .map(|i| start * 10f64.powf(i as f64 / per_decade)) + .filter(|&f| f <= stop * 1.0001) + .collect() + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn arm(&mut self) { + if self.worker.is_some() { + return; + } + match open_transport(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + self.state = RunState::Armed; + self.last_error = None; + } + Err(err) => self.last_error = Some(err), + } + self.bump(); + } + + fn start_run(&mut self) { + if self.worker.is_none() { + self.last_error = Some("run: arm the controller first".into()); + return; + } + self.run_id = format!( + "A1-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + ); + let pdq_path = run_data_dir().join(format!("{}.pdq", self.run_id)); + match PdqWriter::create(&pdq_path) { + Ok(writer) => self.pdq = Some(writer), + Err(err) => { + self.last_error = Some(format!("pdq: {err}")); + return; + } + } + let plan = SweepPlan { + frequencies_hz: self.frequency_grid(), + initial_amplitude_dac: self.initial_amplitude_dac.clamp(1, 2_047) as u32, + ..SweepPlan::default() + }; + self.engine = Some(SweepEngine::new(plan)); + self.reference_counts.clear(); + self.hot_pixels = None; + self.window.clear(); + self.settle_until_us = None; + // Reference phase: unmodulated field (amplitude 0) for the + // hot-pixel mask and the background sanity check. + self.send_drive(self.freq_start_hz, 0, "reference"); + self.state = RunState::Reference; + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(true)); + } + self.bump(); + } + + fn stop_run(&mut self, reason: &str) { + self.queue_command("stop", Command::new("STOP").field("reason", reason)); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + self.finish_run(); + self.state = if self.worker.is_some() { + RunState::Armed + } else { + RunState::Idle + }; + self.bump(); + } + + fn disarm(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + worker.shutdown(reason); + } + self.finish_run(); + self.in_flight.clear(); + self.state = RunState::Idle; + self.bump(); + } + + fn finish_run(&mut self) { + if let Some(pdq) = self.pdq.take() { + match pdq.finish(self.integrity) { + Ok(summary) => { + let mut sidecar = RunSidecar::from_pdq(&self.run_id, "A1", &summary); + sidecar.plugin_name = "stage-a-a1".into(); + sidecar.plugin_version = env!("CARGO_PKG_VERSION").into(); + sidecar.firmware_version = self.firmware.clone(); + sidecar.adc_calibration = self.calibration.clone(); + sidecar.configured_sample_rate_hz = self.sample_rate_hz as u32; + sidecar.trigger_source = if self.used_hardware_fiducial { + TriggerSource::DrivePhase0 + } else { + TriggerSource::None + }; + sidecar.valid = summary.valid; + if let Some(mask) = &self.hot_pixels { + sidecar + .notes + .push(format!("hot pixels masked: {}", mask.masked_count())); + } + let sidecar_path = run_data_dir().join(format!("{}.json", self.run_id)); + if let Err(err) = sidecar.write_json(&sidecar_path) { + self.last_error = Some(format!("sidecar: {err}")); + } + if let Some(engine) = &self.engine { + let results_path = + run_data_dir().join(format!("{}.results.json", self.run_id)); + let _ = std::fs::write( + &results_path, + serde_json::to_vec_pretty(&results_json(engine)).unwrap_or_default(), + ); + } + } + Err(err) => self.last_error = Some(format!("pdq finish: {err}")), + } + } + } + + fn send_drive(&mut self, frequency_hz: f64, amplitude_dac: u32, purpose: &str) { + let freq_mhz = (frequency_hz * 1_000.0).round() as i64; + self.queue_command( + purpose, + Command::new("CONFIG") + .field("mode", "A1") + .field("wave", "SINE") + .field("freq_mhz", freq_mhz) + .field("center_dac", 2_048) + .field("amplitude_dac", amplitude_dac) + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", 256) + .field("raw", 1) + .field("summary", 1), + ); + self.queue_command("start", Command::new("START")); + self.window.clear(); + self.settle_until_us = None; // set on the first camera frame seen + self.current_phase_histogram = None; + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + let mut stopped = None; + for output in outputs { + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => { + if purpose == "hello" { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + } + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + WorkerOutput::Event(DeviceEvent::Data(frame)) => { + if let Some(pdq) = &mut self.pdq { + let _ = pdq.write_frame(&frame); + } + if frame.header.frame_type == FrameType::SamplesU16 { + if let Some(codes) = frame.samples() { + self.window.adc_codes.extend_from_slice(&codes); + } + } + } + WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Integrity(integrity) => self.integrity = integrity, + WorkerOutput::Stopped { reason } => stopped = Some(reason), + } + } + if let Some(reason) = stopped { + self.worker = None; + self.last_error = Some(format!("device connection ended: {reason}")); + self.finish_run(); + self.state = RunState::Idle; + self.bump(); + } + } + + fn ingest_camera_frame(&mut self, frame: &PluginFrame<'_>) { + self.sensor_size = (frame.width(), frame.height()); + self.window.latest_camera_ts_us = frame.window_end_us(); + if self.settle_until_us.is_none() { + self.settle_until_us = + Some(frame.window_end_us() + (self.settle_ms.max(0) as u64) * 1_000); + return; + } + let settle_until = self.settle_until_us.unwrap_or(0); + if frame.window_end_us() < settle_until { + return; + } + self.window + .window_start_us + .get_or_insert(frame.window_start_us()); + + if self.state == RunState::Reference { + if self.reference_counts.len() + != frame.width() as usize * frame.height() as usize + { + self.reference_counts = + vec![0; frame.width() as usize * frame.height() as usize]; + } + for event in frame.events() { + let idx = event.y as usize * frame.width() as usize + event.x as usize; + if let Some(slot) = self.reference_counts.get_mut(idx) { + *slot += 1; + } + } + } else { + let mask = self.hot_pixels.as_ref(); + for event in frame.events() { + if event.is_on() != self.polarity_on { + continue; + } + if mask.is_some_and(|m| m.is_masked(event.x, event.y)) { + continue; + } + self.window.event_timestamps_us.push(event.timestamp_us()); + } + } + for trigger in frame.external_triggers() { + if trigger.is_rising() { + self.window.trigger_edges_us.push(trigger.timestamp_us); + } + } + } + + fn window_target_us(&self, frequency_hz: f64) -> u64 { + ((self.cycles_per_measurement.max(10) as f64 / frequency_hz) * 1.0e6) as u64 + } + + fn advance_run(&mut self) { + match self.state { + RunState::Reference => { + // A fixed 0.5 s of unmodulated reference. + if self.window.elapsed_us() < 500_000 { + return; + } + let (width, height) = self.sensor_size; + if width > 0 && !self.reference_counts.is_empty() { + self.hot_pixels = Some(HotPixelMask::from_reference_counts( + width, + height, + &self.reference_counts, + )); + } + self.state = RunState::Sweeping; + let Some(engine) = &self.engine else { + return; + }; + if let SweepCommand::Measure { + frequency_hz, + amplitude_dac, + } = engine.current_command() + { + self.send_drive(frequency_hz, amplitude_dac, "sweep"); + } + self.bump(); + } + RunState::Sweeping => { + let Some(engine) = &self.engine else { + return; + }; + let SweepCommand::Measure { + frequency_hz, + amplitude_dac, + } = engine.current_command() + else { + self.state = RunState::Finished; + self.finish_run(); + self.bump(); + return; + }; + if self.window.elapsed_us() < self.window_target_us(frequency_hz) { + return; + } + let measurement = self.evaluate_window(frequency_hz, amplitude_dac); + let next = { + let engine = self.engine.as_mut().expect("engine exists"); + engine.ingest(measurement) + }; + match next { + SweepCommand::Measure { + frequency_hz, + amplitude_dac, + } => self.send_drive(frequency_hz, amplitude_dac, "sweep"), + SweepCommand::Finished => { + self.queue_command("stop", Command::new("STOP").field("reason", "done")); + self.state = RunState::Finished; + self.finish_run(); + } + } + self.bump(); + } + _ => {} + } + } + + fn evaluate_window(&mut self, frequency_hz: f64, amplitude_dac: u32) -> Measurement { + // Optical contrast from the photodiode trace; any estimator + // rejection or stream fault invalidates the point. + let measured_a = if self.integrity.is_clean() { + estimate_contrast(&self.window.adc_codes, &self.calibration) + .ok() + .map(|estimate| estimate.a) + } else { + None + }; + + let events = &self.window.event_timestamps_us; + let observed_cycles = self.window.elapsed_us() as f64 / 1.0e6 * frequency_hz; + + // Cycle fiducial: hardware phase-0 edges when present, otherwise + // software frequency refinement against the events themselves. + let (phases, trials) = if self.window.trigger_edges_us.len() >= 2 { + self.used_hardware_fiducial = true; + ( + fold_phases_with_fiducials(events, &self.window.trigger_edges_us), + 1, + ) + } else if let Some(lock) = refine_frequency(events, frequency_hz, 100.0) { + ( + fold_phases( + events.iter().copied(), + events.first().copied().unwrap_or(0), + lock.frequency_hz, + ), + lock.trials, + ) + } else { + (Vec::new(), 1) + }; + + let stat = rayleigh_test(&phases); + let histogram = phase_histogram(&phases, PHASE_BINS); + let excess = phase_locked_excess(&histogram); + self.current_phase_histogram = Some(histogram); + let verdict = detect(stat, excess, observed_cycles, self.alpha, trials); + + Measurement { + amplitude_dac, + measured_a, + events_per_half_cycle: verdict.locked_events_per_cycle, + detected: verdict.detected, + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) + || !request.action_id.starts_with("stage-a-a1.") + { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } + + // -- datasets -------------------------------------------------------- + + fn amin_dataset(&self) -> Series1dV1 { + let mut a_min = Vec::new(); + let mut low = Vec::new(); + let mut high = Vec::new(); + if let Some(engine) = &self.engine { + for result in &engine.results { + if let Some(fit) = &result.fit { + a_min.push(Series1dPoint { + x: result.frequency_hz, + y: fit.a_min, + }); + low.push(Series1dPoint { + x: result.frequency_hz, + y: fit.a_min_low, + }); + high.push(Series1dPoint { + x: result.frequency_hz, + y: fit.a_min_high, + }); + } + } + } + Series1dV1 { + x_label: "drive frequency [Hz]".into(), + y_label: "a_min".into(), + lines: vec![ + Series1dLine { + name: "a_min".into(), + points: a_min, + }, + Series1dLine { + name: "CI low".into(), + points: low, + }, + Series1dLine { + name: "CI high".into(), + points: high, + }, + ], + } + } + + fn phase_dataset(&self) -> Series1dV1 { + let points = self + .current_phase_histogram + .as_ref() + .map(|histogram| { + histogram + .bins + .iter() + .enumerate() + .map(|(i, &count)| Series1dPoint { + x: (i as f64 + 0.5) / histogram.bins.len() as f64, + y: f64::from(count), + }) + .collect() + }) + .unwrap_or_default(); + Series1dV1 { + x_label: "drive phase [cycles]".into(), + y_label: "events".into(), + lines: vec![Series1dLine { + name: if self.polarity_on { "ON" } else { "OFF" }.into(), + points, + }], + } + } + + fn depth_dataset(&self) -> Series1dV1 { + let mut points: Vec = self + .engine + .as_ref() + .map(|engine| { + let mut all: Vec = engine + .results + .last() + .map(|result| { + result + .points + .iter() + .map(|p| Series1dPoint { + x: p.a, + y: p.events_per_half_cycle, + }) + .collect() + }) + .unwrap_or_default(); + all.sort_by(|p, q| p.x.total_cmp(&q.x)); + all + }) + .unwrap_or_default(); + points.dedup_by(|p, q| p.x == q.x); + Series1dV1 { + x_label: "measured a".into(), + y_label: "locked events / half-cycle".into(), + lines: vec![Series1dLine { + name: "N(a)".into(), + points, + }], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("progress", "Progress"), + column("fiducial", "Cycle fiducial"), + column("hot_pixels", "Hot pixels"), + column("integrity", "Integrity"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.state) { + (Some(reason), _) => format!("locked ({reason})"), + (None, RunState::Idle) => "idle".into(), + (None, RunState::Armed) => format!("armed ({})", self.firmware), + (None, RunState::Reference) => "reference window (hot-pixel mask)".into(), + (None, RunState::Sweeping) => "sweeping".into(), + (None, RunState::Finished) => "finished".into(), + }; + let progress = self + .engine + .as_ref() + .map(|engine| { + format!( + "{}/{} frequencies", + engine.results.len(), + engine.results.len() + + if engine.is_finished() { 0 } else { 1 } + ) + }) + .unwrap_or_else(|| "—".into()); + let fiducial = if self.used_hardware_fiducial { + "EXT_TRIGGER phase-0".to_owned() + } else { + "software frequency lock".to_owned() + }; + let hot = self + .hot_pixels + .as_ref() + .map(|mask| format!("{} masked", mask.masked_count())) + .unwrap_or_else(|| "—".into()); + let integrity = if self.integrity.is_clean() { + "clean".to_owned() + } else { + format!( + "crc={} gaps={} overruns={}", + self.integrity.crc_failures, + self.integrity.sequence_gaps, + self.integrity.dropped_samples + ) + }; + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("progress", progress), + text_column("fiducial", fiducial), + text_column("hot_pixels", hot), + text_column("integrity", integrity), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } +} + +fn results_json(engine: &SweepEngine) -> Value { + json!({ + "results": engine + .results + .iter() + .map(|result| { + json!({ + "frequency_hz": result.frequency_hz, + "exhausted": result.exhausted, + "measurements": result.measurements, + "fit": result.fit.as_ref().map(|fit| json!({ + "a_min": fit.a_min, + "a_min_low": fit.a_min_low, + "a_min_high": fit.a_min_high, + "sigma_ln_a": fit.sigma_ln_a, + "points_used": fit.points_used, + })), + "points": result + .points + .iter() + .map(|p| json!({"a": p.a, "events_per_half_cycle": p.events_per_half_cycle})) + .collect::>(), + }) + }) + .collect::>(), + }) +} + +fn run_data_dir() -> PathBuf { + let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_default(); + home.join(".augur").join("stage-a-runs") +} + +fn open_transport(port_hint: &str) -> Result, String> { + let path = if port_hint == "auto" { + stage_a_io::transport::available_port_names() + .into_iter() + .find(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .ok_or_else(|| "no USB serial device found".to_owned())? + } else { + port_hint.to_owned() + }; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +impl Plugin for StageAA1Plugin { + fn name(&self) -> &'static str { + "Stage-A A1 Min-Depth" + } + + fn description(&self) -> &'static str { + "Event-native Bode calibration: a_min(f) via phase-locked detection, bisection, and probit fitting." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disarm("plugin disabled"); + } + } + + fn reset(&mut self) { + self.window.clear(); + self.current_phase_histogram = None; + self.bump(); + } + + fn input_kind(&self) -> PluginInput { + PluginInput::RawEvents + } + + fn process_frame( + &mut self, + frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = + Some(format!("effects not allowed in {:?}", execution.mode)); + if self.worker.is_some() { + self.disarm("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for action in self.consume_actions(context) { + match action.as_str() { + ACTION_ARM => self.arm(), + ACTION_RUN => self.start_run(), + ACTION_STOP => self.stop_run("operator"), + _ => {} + } + } + + self.drain_worker(); + if matches!(self.state, RunState::Reference | RunState::Sweeping) { + self.ingest_camera_frame(frame); + self.advance_run(); + } + } + + fn settings_schema(&self) -> SettingsSchema { + SettingsSchema { + sections: vec![ + SettingsSection { + label: "Sweep".into(), + description: Some( + "Frequency grid and statistics. ON and OFF are measured in separate \ + runs — never pooled." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "freq_start_hz".into(), + label: "Start frequency".into(), + tooltip: None, + kind: SettingKind::F64Drag { + min: 1.0, + max: 1.0e6, + speed: 10.0, + default: self.freq_start_hz, + }, + }, + SettingItem { + key: "freq_stop_hz".into(), + label: "Stop frequency".into(), + tooltip: None, + kind: SettingKind::F64Drag { + min: 1.0, + max: 1.0e6, + speed: 100.0, + default: self.freq_stop_hz, + }, + }, + SettingItem { + key: "points_per_decade".into(), + label: "Points per decade".into(), + tooltip: None, + kind: SettingKind::I64Slider { + min: 2, + max: 12, + default: self.points_per_decade, + suffix: None, + }, + }, + SettingItem { + key: "cycles_per_measurement".into(), + label: "Cycles per measurement".into(), + tooltip: Some( + "Modulation cycles integrated per amplitude point".into(), + ), + kind: SettingKind::I64Slider { + min: 50, + max: 5_000, + default: self.cycles_per_measurement, + suffix: None, + }, + }, + SettingItem { + key: "polarity_on".into(), + label: "Polarity".into(), + tooltip: Some("Which comparator path this sweep measures".into()), + kind: SettingKind::Enum { + variants: vec!["ON".into(), "OFF".into()], + default: usize::from(!self.polarity_on), + }, + }, + SettingItem { + key: "alpha".into(), + label: "Significance α".into(), + tooltip: Some( + "Per-measurement false-positive budget (Bonferroni-corrected \ + for the frequency scan)" + .into(), + ), + kind: SettingKind::F64Drag { + min: 1e-6, + max: 0.05, + speed: 1e-4, + default: self.alpha, + }, + }, + ], + }, + SettingsSection { + label: "Device".into(), + description: None, + default_open: false, + items: vec![ + SettingItem { + key: "initial_amplitude_dac".into(), + label: "Initial amplitude (DAC)".into(), + tooltip: None, + kind: SettingKind::I64Slider { + min: 1, + max: 2_047, + default: self.initial_amplitude_dac, + suffix: None, + }, + }, + SettingItem { + key: "settle_ms".into(), + label: "Settle time".into(), + tooltip: Some( + "Discarded after each drive change (HVA/Pockels settling + \ + refractory clearing)" + .into(), + ), + kind: SettingKind::I64Slider { + min: 10, + max: 2_000, + default: self.settle_ms, + suffix: Some(" ms".into()), + }, + }, + SettingItem { + key: "dark_millivolts".into(), + label: "Dark level".into(), + tooltip: None, + kind: SettingKind::F64Drag { + min: 0.0, + max: 3_300.0, + speed: 1.0, + default: self.calibration.dark_volts * 1_000.0, + }, + }, + ], + }, + ], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "freq_start_hz" => Some(json!(self.freq_start_hz)), + "freq_stop_hz" => Some(json!(self.freq_stop_hz)), + "points_per_decade" => Some(json!(self.points_per_decade)), + "cycles_per_measurement" => Some(json!(self.cycles_per_measurement)), + "polarity_on" => Some(json!(if self.polarity_on { "ON" } else { "OFF" })), + "alpha" => Some(json!(self.alpha)), + "initial_amplitude_dac" => Some(json!(self.initial_amplitude_dac)), + "settle_ms" => Some(json!(self.settle_ms)), + "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "freq_start_hz" => { + self.freq_start_hz = value.as_f64().ok_or("must be a number")?.max(1.0); + } + "freq_stop_hz" => { + self.freq_stop_hz = value.as_f64().ok_or("must be a number")?.max(1.0); + } + "points_per_decade" => { + self.points_per_decade = value.as_i64().ok_or("must be an integer")?.clamp(2, 12); + } + "cycles_per_measurement" => { + self.cycles_per_measurement = + value.as_i64().ok_or("must be an integer")?.clamp(50, 5_000); + } + "polarity_on" => { + let text = value.as_str().ok_or("must be a string")?; + self.polarity_on = text.eq_ignore_ascii_case("on"); + } + "alpha" => { + self.alpha = value.as_f64().ok_or("must be a number")?.clamp(1e-6, 0.05); + } + "initial_amplitude_dac" => { + self.initial_amplitude_dac = + value.as_i64().ok_or("must be an integer")?.clamp(1, 2_047); + } + "settle_ms" => { + self.settle_ms = value.as_i64().ok_or("must be an integer")?.clamp(10, 2_000); + } + "dark_millivolts" => { + let mv = value.as_f64().ok_or("must be a number")?; + self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); + } + _ => return Err(format!("unknown setting: {key}")), + } + Ok(()) + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + if let Some(engine) = &self.engine { + entries.push(StatusEntry::Text(format!( + "{} frequency points finished", + engine.results.len() + ))); + } + if let Some(err) = &self.last_error { + entries.push(StatusEntry::Text(format!("Error: {err}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: AMIN_DATASET_ID.into(), + title: "a_min(f)".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No fitted frequency points yet.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: PHASE_DATASET_ID.into(), + title: "Phase histogram".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No measurement window yet.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: DEPTH_DATASET_ID.into(), + title: "N(a) at current frequency".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No depth points yet.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "A1 run status".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: format!("{AMIN_DATASET_ID}.view"), + title: "A1 Bode (a_min)".into(), + dataset_id: AMIN_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: format!("{PHASE_DATASET_ID}.view"), + title: "Phase fold".into(), + dataset_id: PHASE_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: format!("{DEPTH_DATASET_ID}.view"), + title: "Depth staircase".into(), + dataset_id: DEPTH_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: format!("{STATUS_DATASET_ID}.view"), + title: "A1 status".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + HostActionDescriptor { + id: ACTION_ARM.into(), + title: "Arm controller".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_RUN.into(), + title: "Run A1 sweep".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_STOP.into(), + title: "Stop".into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }, + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + AMIN_DATASET_ID => serde_json::to_vec(&self.amin_dataset()).ok(), + PHASE_DATASET_ID => serde_json::to_vec(&self.phase_dataset()).ok(), + DEPTH_DATASET_ID => serde_json::to_vec(&self.depth_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + AMIN_DATASET_ID | PHASE_DATASET_ID | DEPTH_DATASET_ID | STATUS_DATASET_ID => { + self.dataset_generation.max(1) + } + _ => 0, + } + } +} + +impl Drop for StageAA1Plugin { + fn drop(&mut self) { + self.disarm("plugin destroyed"); + } +} + +export_plugin!(StageAA1Plugin); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frequency_grid_is_log_spaced_and_bounded() { + let mut plugin = StageAA1Plugin::default(); + plugin.freq_start_hz = 100.0; + plugin.freq_stop_hz = 10_000.0; + plugin.points_per_decade = 4; + let grid = plugin.frequency_grid(); + assert!((grid.first().copied().unwrap() - 100.0).abs() < 1e-9); + assert!(grid.last().copied().unwrap() <= 10_000.0 * 1.001); + assert_eq!(grid.len(), 9); + for pair in grid.windows(2) { + let ratio = pair[1] / pair[0]; + assert!((ratio - 10f64.powf(0.25)).abs() < 1e-9); + } + } + + #[test] + fn status_dataset_matches_schema() { + let plugin = StageAA1Plugin::default(); + let dataset = plugin.status_dataset(); + let schema = plugin.status_schema(); + assert_eq!(dataset.columns.len(), schema.columns.len()); + } +} diff --git a/plugins/stage-a-a1/src/sweep.rs b/plugins/stage-a-a1/src/sweep.rs new file mode 100644 index 0000000..1f48cc8 --- /dev/null +++ b/plugins/stage-a-a1/src/sweep.rs @@ -0,0 +1,363 @@ +//! Minimum-depth sweep state machine. +//! +//! For each frequency point: bisect on the integer DAC drive code until the +//! detection boundary is bracketed, then measure a small log-spaced grid +//! across the transition, then fit `a_min` (see `analysis::fit_min_depth`). +//! The engine is pure — device I/O and event analysis happen outside; it +//! only ingests finished measurements and emits the next drive request. +//! Note the asymmetry the whole design hinges on: the *search* variable is +//! the drive code, but every recorded point carries the **measured** +//! optical contrast `a` from the photodiode. + +use crate::analysis::{fit_min_depth, DepthPoint, MinDepthFit}; + +#[derive(Debug, Clone, PartialEq)] +pub struct SweepPlan { + pub frequencies_hz: Vec, + pub initial_amplitude_dac: u32, + pub max_amplitude_dac: u32, + /// Grid points measured across the bracket after bisection. + pub grid_points: usize, + /// Hard cap on measurements per frequency (bisection + grid). + pub max_measurements_per_frequency: usize, +} + +impl Default for SweepPlan { + fn default() -> Self { + Self { + frequencies_hz: Vec::new(), + initial_amplitude_dac: 512, + max_amplitude_dac: 2_047, + grid_points: 6, + max_measurements_per_frequency: 24, + } + } +} + +/// One finished measurement at the currently requested drive. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Measurement { + pub amplitude_dac: u32, + /// Photodiode-measured optical log-contrast. `None` = invalid window + /// (clipped / integrity fault) — the point is discarded and re-measured. + pub measured_a: Option, + pub events_per_half_cycle: f64, + pub detected: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum SweepCommand { + /// Configure the drive and measure at these settings. + Measure { frequency_hz: f64, amplitude_dac: u32 }, + /// All frequencies finished. + Finished, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FrequencyResult { + pub frequency_hz: f64, + pub fit: Option, + pub points: Vec, + pub measurements: usize, + /// True when the point budget ran out before the transition was + /// bracketed — a_min is not identifiable from this data. + pub exhausted: bool, +} + +#[derive(Debug, Clone, PartialEq)] +enum Phase { + Bisecting, + Grid { queue: Vec }, +} + +pub struct SweepEngine { + plan: SweepPlan, + frequency_index: usize, + phase: Phase, + current_dac: u32, + measurements_at_frequency: usize, + /// Highest drive code that did NOT detect / lowest that did. + highest_undetected: Option, + lowest_detected: Option, + points: Vec, + invalid_retries: usize, + pub results: Vec, +} + +impl SweepEngine { + pub fn new(plan: SweepPlan) -> Self { + let current_dac = plan.initial_amplitude_dac; + Self { + plan, + frequency_index: 0, + phase: Phase::Bisecting, + current_dac, + measurements_at_frequency: 0, + highest_undetected: None, + lowest_detected: None, + points: Vec::new(), + invalid_retries: 0, + results: Vec::new(), + } + } + + pub fn current_command(&self) -> SweepCommand { + match self.plan.frequencies_hz.get(self.frequency_index) { + Some(&frequency_hz) => SweepCommand::Measure { + frequency_hz, + amplitude_dac: self.current_dac, + }, + None => SweepCommand::Finished, + } + } + + pub fn is_finished(&self) -> bool { + self.frequency_index >= self.plan.frequencies_hz.len() + } + + /// Ingests the finished measurement for the last `Measure` command and + /// advances the state machine. + pub fn ingest(&mut self, measurement: Measurement) -> SweepCommand { + if self.is_finished() { + return SweepCommand::Finished; + } + + let Some(a) = measurement.measured_a else { + // Invalid window: re-measure the same point (bounded retries), + // never silently keep the previous contrast. + self.invalid_retries += 1; + if self.invalid_retries > 3 { + self.finish_frequency(true); + } + return self.current_command(); + }; + self.invalid_retries = 0; + self.measurements_at_frequency += 1; + self.points.push(DepthPoint { + a, + events_per_half_cycle: measurement.events_per_half_cycle, + }); + + if measurement.detected { + self.lowest_detected = Some( + self.lowest_detected + .map_or(measurement.amplitude_dac, |d| d.min(measurement.amplitude_dac)), + ); + } else { + self.highest_undetected = Some( + self.highest_undetected + .map_or(measurement.amplitude_dac, |d| d.max(measurement.amplitude_dac)), + ); + } + + if self.measurements_at_frequency >= self.plan.max_measurements_per_frequency { + self.finish_frequency(!self.bracketed()); + return self.current_command(); + } + + match &mut self.phase { + Phase::Bisecting => { + if self.bracketed() { + let queue = self.grid_queue(); + self.phase = Phase::Grid { queue }; + self.advance_grid(); + } else if measurement.detected { + // Drive down toward the boundary. + let next = ((measurement.amplitude_dac as f64) * 0.65).round() as u32; + if next < 1 { + self.finish_frequency(false); + } else { + self.current_dac = next.max(1); + } + } else { + // Drive up toward the boundary. + let next = ((measurement.amplitude_dac as f64) * 1.5).ceil() as u32; + if next > self.plan.max_amplitude_dac { + // Even full drive shows nothing: unmeasurable point. + self.finish_frequency(true); + } else { + self.current_dac = next; + } + } + } + Phase::Grid { .. } => { + self.advance_grid(); + } + } + self.current_command() + } + + fn bracketed(&self) -> bool { + matches!( + (self.highest_undetected, self.lowest_detected), + (Some(_), Some(_)) + ) + } + + fn grid_queue(&self) -> Vec { + let (Some(low), Some(high)) = (self.highest_undetected, self.lowest_detected) else { + return Vec::new(); + }; + let lo = (low.min(high) as f64 * 0.8).max(1.0); + let hi = (low.max(high) as f64 * 1.25).min(self.plan.max_amplitude_dac as f64); + let n = self.plan.grid_points.max(2); + (0..n) + .map(|i| { + let t = i as f64 / (n - 1) as f64; + (lo * (hi / lo).powf(t)).round() as u32 + }) + .collect() + } + + fn advance_grid(&mut self) { + let next = match &mut self.phase { + Phase::Grid { queue } if !queue.is_empty() => Some(queue.remove(0)), + _ => None, + }; + match next { + Some(dac) => self.current_dac = dac, + None => self.finish_frequency(false), + } + } + + fn finish_frequency(&mut self, exhausted: bool) { + let frequency_hz = self.plan.frequencies_hz[self.frequency_index]; + let fit = if exhausted { + None + } else { + fit_min_depth(&self.points) + }; + self.results.push(FrequencyResult { + frequency_hz, + fit, + points: std::mem::take(&mut self.points), + measurements: self.measurements_at_frequency, + exhausted, + }); + self.frequency_index += 1; + self.phase = Phase::Bisecting; + self.current_dac = self.plan.initial_amplitude_dac; + self.measurements_at_frequency = 0; + self.highest_undetected = None; + self.lowest_detected = None; + self.invalid_retries = 0; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Simulated bench: optical contrast is proportional to the drive code + /// (a = dac / 2000) and the pixel responds with the smeared first step + /// around a_min = 0.2. + fn respond(dac: u32) -> Measurement { + let a = dac as f64 / 2_000.0; + let z = (a.ln() - 0.2_f64.ln()) / 0.12; + let n = 0.5 * (1.0 + erf_approx(z / std::f64::consts::SQRT_2)); + Measurement { + amplitude_dac: dac, + measured_a: Some(a), + events_per_half_cycle: n, + detected: n > 0.15, + } + } + + fn erf_approx(x: f64) -> f64 { + let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); + let poly = t + * (0.254_829_592 + + t * (-0.284_496_736 + + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); + let value = 1.0 - poly * (-x * x).exp(); + if x >= 0.0 { + value + } else { + -value + } + } + + #[test] + fn converges_to_the_synthetic_a_min() { + let mut engine = SweepEngine::new(SweepPlan { + frequencies_hz: vec![1_000.0, 10_000.0], + ..SweepPlan::default() + }); + + let mut guard = 0; + loop { + guard += 1; + assert!(guard < 200, "sweep must terminate"); + match engine.current_command() { + SweepCommand::Finished => break, + SweepCommand::Measure { amplitude_dac, .. } => { + engine.ingest(respond(amplitude_dac)); + } + } + } + + assert_eq!(engine.results.len(), 2); + for result in &engine.results { + let fit = result.fit.as_ref().expect("fit must exist"); + assert!( + (fit.a_min - 0.2).abs() < 0.04, + "f={} a_min={}", + result.frequency_hz, + fit.a_min + ); + assert!(!result.exhausted); + } + } + + #[test] + fn undetectable_frequency_is_reported_exhausted_not_fitted() { + let mut engine = SweepEngine::new(SweepPlan { + frequencies_hz: vec![100_000.0], + ..SweepPlan::default() + }); + let mut guard = 0; + loop { + guard += 1; + assert!(guard < 100); + match engine.current_command() { + SweepCommand::Finished => break, + SweepCommand::Measure { amplitude_dac, .. } => { + engine.ingest(Measurement { + amplitude_dac, + measured_a: Some(amplitude_dac as f64 / 2_000.0), + events_per_half_cycle: 0.0, + detected: false, + }); + } + } + } + assert_eq!(engine.results.len(), 1); + assert!(engine.results[0].exhausted); + assert!(engine.results[0].fit.is_none()); + } + + #[test] + fn invalid_windows_are_retried_then_abandoned() { + let mut engine = SweepEngine::new(SweepPlan { + frequencies_hz: vec![1_000.0], + ..SweepPlan::default() + }); + let mut measures = 0; + loop { + match engine.current_command() { + SweepCommand::Finished => break, + SweepCommand::Measure { amplitude_dac, .. } => { + measures += 1; + assert!(measures < 20); + engine.ingest(Measurement { + amplitude_dac, + measured_a: None, + events_per_half_cycle: 0.0, + detected: false, + }); + } + } + } + assert!(engine.results[0].exhausted); + } +} From 77e859a6ec5e54d7c000dc16611c78667ea56ba7 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 17:21:31 +0200 Subject: [PATCH 04/30] =?UTF-8?q?docs(stage-a):=20=F0=9F=93=9D=20add=20Sta?= =?UTF-8?q?ge-A=20feature=20brief=20and=20device-ownership=20ADR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature brief for the stage-a-io/monitor/a1 stack, ADR 005 for the device-ownership boundary (plugins own the Teensy, AugurRs stays generic), feature index entry. Note: the legacy plugins (localization/reconstruction/focus-metrics/ evesmlm) on this branch predate the current augur-rs plugin API and do not compile against it — their refresh is in progress on feature/eve-batch-findings; rebasing that work onto plugin ABI v5 only adds the new FfiPreviewFrame.external_triggers field in one test initializer. --- docs/adr/005-stage-a-device-ownership.md | 45 +++++++++++++++ docs/features/README.md | 1 + docs/features/stage-a.md | 72 ++++++++++++++++++++++++ plugins/stage-a-a1/src/analysis.rs | 17 ++++-- plugins/stage-a-a1/src/lib.rs | 14 ++--- plugins/stage-a-a1/src/sweep.rs | 17 ++++-- plugins/stage-a-monitor/src/lib.rs | 22 ++++---- 7 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 docs/adr/005-stage-a-device-ownership.md create mode 100644 docs/features/stage-a.md diff --git a/docs/adr/005-stage-a-device-ownership.md b/docs/adr/005-stage-a-device-ownership.md new file mode 100644 index 0000000..57bcec5 --- /dev/null +++ b/docs/adr/005-stage-a-device-ownership.md @@ -0,0 +1,45 @@ +# ADR 005 — Stage-A device ownership and the `stage-a-io` boundary + +- **Status:** Accepted +- **Date:** 2026-07-13 + +## Context + +The Stage-A camera calibrations (A1–A3) drive a Teensy stimulus/DAQ +controller over USB serial while recording the event camera. Someone has +to own the serial port, the experiment state machines, and the safety +rules. The knowledge-base control-software spec fixes the boundary: +AugurRs stays a generic camera recorder and plugin host and must not gain +laboratory-instrument abstractions. + +## Decision + +1. **Device control lives in removable protocol plugins** (`stage-a-monitor`, + `stage-a-a1`, later `-a2`/`-a3`), one experiment concern per plugin. + Exactly one enabled, armed plugin owns the serial port; opening a busy + device is a visible error. +2. **A shared plain-Rust library `stage-a-io`** (this repo, not a plugin) + owns everything protocol-shaped: PDA1 framing + CRC resync, the ASCII + command grammar with idempotent sequence retries, the bounded I/O + worker, `.pdq` persistence, the run sidecar, and the calibrated optical + contrast estimator. It contains **no experiment policy** (sweeps, + bisection, fits stay in the plugins) and **no augur types** (testable + without a host). +3. **Effects are gated by the host's execution context** (plugin ABI v5): + plugins fail closed unless `LiveCapture && effects_allowed`. Hardware + commands are host actions, never persistent settings. +4. **Wire compatibility is anchored to the firmware header** + (`stage-a-controller/include/wire_protocol.h`); `stage-a-io` mirrors it + with layout tests, and the mock controller implements the same + idempotency contract the firmware promises. + +## Consequences + +- A2/A3 plugins reuse `stage-a-io` unchanged; only their state machines + and views are new code. +- The GUI knows nothing about Teensys; removing the three plugins removes + every trace of lab hardware from the product. +- Protocol changes must land in the firmware header first, then in + `stage-a-io`, keeping a single source of truth for the wire format. +- Plugins depend on `stage-a-io` by path; it is versioned with the + workspace and its API may still move until A2/A3 land. diff --git a/docs/features/README.md b/docs/features/README.md index b39d965..7127a0a 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -4,6 +4,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs +- [Stage-A Calibration Plugins](./stage-a.md) — Teensy-driven Stage-A bench stack: `stage-a-io` shared I/O, commissioning monitor, and the A1 minimum-depth Bode sweep. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Runtime Migration Notes](./plugin-api-v0-2.md) — historical runtime-migration brief, updated with the current interface additions that matter to this repo. - [Plugin Host Views](./plugin-host-views.md) — generic host-rendered datasets, cache generations, and shared view ids. diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md new file mode 100644 index 0000000..7924c6b --- /dev/null +++ b/docs/features/stage-a.md @@ -0,0 +1,72 @@ +# Stage-A calibration plugins (`stage-a-io`, `stage-a-monitor`, `stage-a-a1`) + +> Feature brief — first delivery of the Stage-A camera-calibration stack. +> Design source of truth: knowledge base +> `methodology/stage-a-control-software.md` and +> `methodology/camera-calibration.md` (A1 protocol). + +## Architecture + +```text +AugurRs generic host (camera, RAW, EXT_TRIGGER delivery, execution context — ABI v5) + │ + ├── stage-a-monitor — commissioning: live photodiode view, manual control + └── stage-a-a1 — A1 minimum-depth a_min(f) sweep + │ (exactly one armed plugin owns the device) + ▼ + stage-a-io (this repo, plain lib) ── USB serial ── Teensy stage-a-controller +``` + +AugurRs itself gains no Teensy or serial abstraction — device ownership +lives entirely in these removable plugins (ADR 005). + +## Crates + +| Crate | Role | +|---|---| +| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, mock controller | +| `plugins/stage-a-monitor` | Live decimated waveform, live `a`, integrity status, gated manual CONFIG/START/STOP + expert drive modal | +| `plugins/stage-a-a1` | Phase-locked detection (Rayleigh), hardware/software cycle fiducials, bisection + grid sweep, probit `a_min` fit with CI, hot-pixel mask, PDQ + sidecar + results export | + +## Safety model + +- Serial ports open only when `HostContext::execution()` reports + `LiveCapture` **and** `effects_allowed` (host constructs this fail-closed; + only the active live-capture worker qualifies). Replay can never re-arm + hardware, even from a sidecar that contains a runnable setup. +- All hardware commands are host **actions**; persistent settings never + start hardware after a reload. +- Any CRC error, frame-sequence gap, or ADC overrun invalidates the + measurement point; invalid points are re-measured, never patched, and + the run sidecar records the counters. +- The firmware watchdog (1.5 s) drops the controller to `SAFE_IDLE` + independently of host-side cleanup. + +## Statistics (A1) + +Detection is a phase-uniformity test (background activity is uniform in +drive phase; signal is phase-locked), with the frequency-scan multiplicity +Bonferroni-charged when the software clock-skew lock substitutes for the +missing trigger cable. `a_min` is the fitted `N = 0.5` crossing of a +probit in `ln a` with a profile CI — not a raw bisection endpoint — and +`a` is always the photodiode-measured contrast. Details and rationale: +`plugins/stage-a-a1/README.md`. + +## Verification + +`cargo test` (38 tests): wire fragmentation/CRC-resync/overrun, retry +idempotency against the mock controller, worker round-trip + clean STOP, +estimator recovery/clipping/headroom guards, Rayleigh calibration on +uniform and locked phases, background-immunity, clock-skew recovery +(300 ppm), fiducial folding, probit fit recovery, sweep convergence to a +synthetic `a_min`, exhaustion/invalid-window handling, hot-pixel masking, +dataset/schema consistency. + +## Known gaps + +- Final Teensy DDS/DAC firmware is blocked on the hardware freeze; the + sweep runs against the v1 protocol and the mock meanwhile. +- Marker cycles are protocol-reserved but not yet emitted + (`stage-a-controller/docs/features/a1-marker-cycles.md`). +- `stage-a-a2` / `stage-a-a3` plugins are not yet implemented; A2 + additionally requires the physical trigger cable. diff --git a/plugins/stage-a-a1/src/analysis.rs b/plugins/stage-a-a1/src/analysis.rs index 32cce7c..0c3ce31 100644 --- a/plugins/stage-a-a1/src/analysis.rs +++ b/plugins/stage-a-a1/src/analysis.rs @@ -112,8 +112,9 @@ pub fn rayleigh_test(phases: &[f64]) -> RayleighResult { let r = (c * c + s * s).sqrt() / n as f64; let z = n as f64 * r * r; let nf = n as f64; - let p = (-z).exp() * (1.0 + (2.0 * z - z * z) / (4.0 * nf) - - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); + let p = (-z).exp() + * (1.0 + (2.0 * z - z * z) / (4.0 * nf) + - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); RayleighResult { n, r, @@ -283,7 +284,8 @@ fn standard_normal_cdf(z: f64) -> f64 { let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); let poly = t * (0.254_829_592 - + t * (-0.284_496_736 + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); + + t * (-0.284_496_736 + + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); let erf_abs = 1.0 - poly * (-x * x).exp(); let erf = if x >= 0.0 { erf_abs } else { -erf_abs }; 0.5 * (1.0 + erf) @@ -297,7 +299,9 @@ pub fn fit_min_depth(points: &[DepthPoint]) -> Option { let usable: Vec = points .iter() .copied() - .filter(|p| p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5) + .filter(|p| { + p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5 + }) .collect(); if usable.len() < 3 { return None; @@ -308,7 +312,10 @@ pub fn fit_min_depth(points: &[DepthPoint]) -> Option { return None; } - let ln_min = usable.iter().map(|p| p.a.ln()).fold(f64::INFINITY, f64::min); + let ln_min = usable + .iter() + .map(|p| p.a.ln()) + .fold(f64::INFINITY, f64::min); let ln_max = usable .iter() .map(|p| p.a.ln()) diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs index c1fd032..379a7f8 100644 --- a/plugins/stage-a-a1/src/lib.rs +++ b/plugins/stage-a-a1/src/lib.rs @@ -403,11 +403,8 @@ impl StageAA1Plugin { .get_or_insert(frame.window_start_us()); if self.state == RunState::Reference { - if self.reference_counts.len() - != frame.width() as usize * frame.height() as usize - { - self.reference_counts = - vec![0; frame.width() as usize * frame.height() as usize]; + if self.reference_counts.len() != frame.width() as usize * frame.height() as usize { + self.reference_counts = vec![0; frame.width() as usize * frame.height() as usize]; } for event in frame.events() { let idx = event.y as usize * frame.width() as usize + event.x as usize; @@ -715,8 +712,7 @@ impl StageAA1Plugin { format!( "{}/{} frequencies", engine.results.len(), - engine.results.len() - + if engine.is_finished() { 0 } else { 1 } + engine.results.len() + if engine.is_finished() { 0 } else { 1 } ) }) .unwrap_or_else(|| "—".into()); @@ -786,7 +782,9 @@ fn results_json(engine: &SweepEngine) -> Value { } fn run_data_dir() -> PathBuf { - let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_default(); + let home = std::env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_default(); home.join(".augur").join("stage-a-runs") } diff --git a/plugins/stage-a-a1/src/sweep.rs b/plugins/stage-a-a1/src/sweep.rs index 1f48cc8..36f0a25 100644 --- a/plugins/stage-a-a1/src/sweep.rs +++ b/plugins/stage-a-a1/src/sweep.rs @@ -48,7 +48,10 @@ pub struct Measurement { #[derive(Debug, Clone, PartialEq)] pub enum SweepCommand { /// Configure the drive and measure at these settings. - Measure { frequency_hz: f64, amplitude_dac: u32 }, + Measure { + frequency_hz: f64, + amplitude_dac: u32, + }, /// All frequencies finished. Finished, } @@ -139,14 +142,16 @@ impl SweepEngine { }); if measurement.detected { - self.lowest_detected = Some( - self.lowest_detected - .map_or(measurement.amplitude_dac, |d| d.min(measurement.amplitude_dac)), - ); + self.lowest_detected = + Some(self.lowest_detected.map_or(measurement.amplitude_dac, |d| { + d.min(measurement.amplitude_dac) + })); } else { self.highest_undetected = Some( self.highest_undetected - .map_or(measurement.amplitude_dac, |d| d.max(measurement.amplitude_dac)), + .map_or(measurement.amplitude_dac, |d| { + d.max(measurement.amplitude_dac) + }), ); } diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs index f01c1f7..ab159aa 100644 --- a/plugins/stage-a-monitor/src/lib.rs +++ b/plugins/stage-a-monitor/src/lib.rs @@ -26,8 +26,8 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, - IoWorker, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, + estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, + StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, }; const WAVEFORM_DATASET_ID: &str = "stage-a-monitor.waveform"; @@ -223,18 +223,16 @@ impl StageAMonitorPlugin { } } } - WorkerOutput::Event(DeviceEvent::Data(frame)) => { - match frame.header.frame_type { - FrameType::SamplesU16 => { - if let Some(codes) = frame.samples() { - self.sample_rate_seen_hz = frame.header.sample_rate_hz; - self.push_samples(&codes, frame.header.first_sample_index); - } + WorkerOutput::Event(DeviceEvent::Data(frame)) => match frame.header.frame_type { + FrameType::SamplesU16 => { + if let Some(codes) = frame.samples() { + self.sample_rate_seen_hz = frame.header.sample_rate_hz; + self.push_samples(&codes, frame.header.first_sample_index); } - FrameType::Summary | FrameType::Marker | FrameType::Control => {} - FrameType::Unknown(_) => {} } - } + FrameType::Summary | FrameType::Marker | FrameType::Control => {} + FrameType::Unknown(_) => {} + }, WorkerOutput::Event(DeviceEvent::Async { .. }) => {} WorkerOutput::Integrity(integrity) => { self.integrity = integrity; From d6c6f78cf02540149e87f446dd388ed103475906 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 22:51:11 +0200 Subject: [PATCH 05/30] =?UTF-8?q?feat(evesmlm,plugins):=20=E2=9C=A8=20batc?= =?UTF-8?q?h=20review=20findings=20and=20current=20plugin-API=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the evesmlm suite, localization, and reconstruction plugins plus repo docs to the current augur-rs plugin interface (host-view actions, dataset display metadata/relations, TableSchema extensions, CompactEvent accessors). --- CONTRIBUTING.md | 20 + README.md | 31 +- .../005-investigation-workspace-datasets.md | 43 + docs/architecture.md | 16 +- docs/features/README.md | 6 + docs/features/action-requests-and-refit.md | 93 + .../features/clickable-overlays-source-row.md | 46 + docs/features/evesmlm-temporal-diagnostics.md | 65 + docs/features/evesmlm.md | 23 +- .../investigation-workspace-alignment.md | 79 + docs/features/plugin-host-views.md | 15 +- docs/features/plugin-install-reload.md | 40 + docs/features/reconstruction.md | 17 +- docs/features/tablev1-declarative-metadata.md | 59 + docs/installing-plugins.md | 20 + docs/plugin-api.md | 221 +- plugin-template/README.md | 9 + plugins/evesmlm-candidates/README.md | 23 +- .../evesmlm-candidates/src/eigenfeature.rs | 37 +- plugins/evesmlm-candidates/src/lib.rs | 1284 ++++++++- plugins/evesmlm-candidates/src/types.rs | 44 +- plugins/evesmlm-fitting/README.md | 18 +- plugins/evesmlm-fitting/src/lib.rs | 2534 +++++++++++++++-- plugins/evesmlm-fitting/src/types.rs | 46 + plugins/evesmlm-postproc/README.md | 13 +- plugins/evesmlm-postproc/src/lib.rs | 71 +- plugins/localization/src/lib.rs | 4 +- plugins/reconstruction/README.md | 5 +- plugins/reconstruction/src/lib.rs | 119 +- scripts/install-built-plugins.sh | 19 +- 30 files changed, 4615 insertions(+), 405 deletions(-) create mode 100644 docs/adr/005-investigation-workspace-datasets.md create mode 100644 docs/features/action-requests-and-refit.md create mode 100644 docs/features/clickable-overlays-source-row.md create mode 100644 docs/features/evesmlm-temporal-diagnostics.md create mode 100644 docs/features/investigation-workspace-alignment.md create mode 100644 docs/features/plugin-install-reload.md create mode 100644 docs/features/tablev1-declarative-metadata.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7edc25d..af41406 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -137,6 +137,18 @@ Plugins do not render `egui` directly. Instead, expose: The host owns rendering, export, caching, and window state for declared host views. +When a table dataset should participate in the linked investigation workspace, also populate the additive metadata the host can use: + +- `coordinate_space_2d` +- `coordinate_space_3d` +- `row_id_column` +- `time_column` +- `layer_id` +- `semantic_label` +- `HostDatasetDescriptor.display` + +Prefer structured datasets for selection/linking and use overlays only for supplemental 2D annotations or hit-testing. + ### 7. Write `plugin.toml` Use the runtime format: @@ -160,6 +172,14 @@ cp plugins/my-plugin/plugin.toml ~/.augur/plugins/my-plugin/ cp target/release/libaugur_plugin_my_plugin.dylib ~/.augur/plugins/my-plugin/ ``` +On macOS, either run `./scripts/install-built-plugins.sh --profile release` instead of the manual +copy steps or rewrite the installed dylib id yourself: + +```bash +install_name_tool -id "@loader_path/libaugur_plugin_my_plugin.dylib" \ + ~/.augur/plugins/my-plugin/libaugur_plugin_my_plugin.dylib +``` + Then open `augur-gui`, go to **Plugins**, click **Scan for New Plugins**, and enable the plugin. ## Migrating Older Plugins diff --git a/README.md b/README.md index 40cb651..4d48dd1 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,27 @@ Use this repository for the plugin implementations, template crate, and repo-loc ## Runtime Model - Each plugin ships as a `plugin.toml` manifest plus one platform library (`.dylib`, `.so`, or `.dll`). -- `augur-gui` discovers plugins from `~/.augur/plugins/`, loads the exported `augur_plugin_vtable`, and renders settings, status, and host views through the host. +- `augur-gui` discovers plugins from `~/.augur/plugins/`, loads the exported `augur_plugin_vtable`, and renders settings, status, and linked investigation datasets/views through the host. - Host-owned built-in tools stay in `augur-gui`; they are not runtime plugins in this repository. - Host-owned experiment settings such as pixel scale, sensor geometry, acquisition time, and EventStore budget are published to plugins as `GlobalSettings` on `augur.global_settings`. - Standard shared scientific payloads can also live in companion crates such as `augur-plugin-types`. +## Investigation Workspace Contract + +The host now owns a generic linked workspace across: + +- 2D preview +- 3D inspection +- host-rendered tables + +For plugins, that means: + +- structured datasets are the primary linking mechanism +- stable row ids should be provided when possible +- 2D/3D coordinate metadata should be declared when the plugin has it +- layer/display metadata should describe visibility, color, marker shape, and size +- overlays are supplemental annotations, not the primary integration surface + ## In-Tree Runtime Plugins (work in progress) The plugin crates under `plugins/` are under active development and not yet ready for external use. The template crate and documentation are stable references for writing your own plugins. @@ -37,11 +53,11 @@ The plugin crates under `plugins/` are under active development and not yet read | Plugin | Phase | Notes | |---|---|---| | `localization` | `RawEvents` | Wavelet/Gaussian SMLM localization and standard `LocalizationResults` output | -| `reconstruction` | `DerivedData` | Accumulated localization table plus host-rendered reconstruction windows | +| `reconstruction` | `DerivedData` | Accumulated localization dataset with stable ids, time metadata, density rendering, and 3D inspection | | `focus-metrics` | `DerivedData` | Focus metrics from localization results or FFT preview sharpness | -| `evesmlm-candidates` | `RawEvents` | Event-domain candidate clustering for eveSMLM | -| `evesmlm-fitting` | `DerivedData` | Candidate fitting plus EVE and compatibility localization outputs | -| `evesmlm-postproc` | `DerivedData` | Filtering, drift correction, evaluation, and the later EVE compact view provider | +| `evesmlm-candidates` | `RawEvents` | Event-domain candidate clustering plus accepted/rejected raw-event investigation layers | +| `evesmlm-fitting` | `DerivedData` | Candidate fitting plus shared current-localization datasets, stable ids, and linked 3D inspection | +| `evesmlm-postproc` | `DerivedData` | Filtering, drift correction, evaluation, and the later shared EVE current-localization provider | `plugin-template/` is the starting point for new plugin crates. @@ -62,6 +78,8 @@ cp target/release/libaugur_plugin_localization.dylib ~/.augur/plugins/localizati ``` On Linux, copy the `.so`. On Windows, copy the `.dll`. +On macOS, prefer `./scripts/install-built-plugins.sh --profile release`; it rewrites the copied +plugin dylib id so Plugin Manager reloads do not keep pointing at Cargo's build tree. Then open `augur-gui`, go to **Plugins**, click **Scan for New Plugins**, and enable the plugin. @@ -101,7 +119,8 @@ The current authoring flow is: 2. export the vtable with `export_plugin!` 3. choose `input_kind()` and optional `PluginCapabilities` 4. use `HostContext` for shared payloads, companion crates such as `augur-plugin-types` for reusable payload types, and `CTX_GLOBAL_SETTINGS` for host-owned calibration/settings -5. declare host-rendered outputs with `host_views()` when needed +5. declare host-rendered outputs with `host_views()` when needed and populate stable-id / coordinate / layer metadata when the dataset should participate in linked investigation + - to expose interactive operations, append `HostActionDescriptor`s to `HostViewRegistry.actions` (scope `Dataset`/`Row`/`Cluster`, optional `param_schema`); consume requests from the persistent context key `CTX_INVESTIGATION_ACTION_REQUESTS` 6. build a `cdylib` 7. install `plugin.toml` plus the compiled library into `~/.augur/plugins//` diff --git a/docs/adr/005-investigation-workspace-datasets.md b/docs/adr/005-investigation-workspace-datasets.md new file mode 100644 index 0000000..03754e7 --- /dev/null +++ b/docs/adr/005-investigation-workspace-datasets.md @@ -0,0 +1,43 @@ +# ADR 005: Expose Generic Investigation Datasets From Plugins + +## Status + +Accepted + +## Context + +`augur-gui` now owns a generic linked investigation workspace across 2D preview, 3D inspection, and host-rendered tables. + +That host model depends on richer plugin-side dataset metadata than the older window-centric host-view integration used: + +- stable row ids +- optional 2D and 3D coordinates +- optional time columns +- layer ids and display metadata + +The eveSMLM pipeline also needs stage-local investigation surfaces for tuning, especially at the candidate-finding stage where researchers need to compare accepted and rejected raw events directly. + +## Decision + +Plugins in this repository will align to the investigation workspace through generic structured datasets. + +Rules: + +1. Use table datasets as the primary linking surface for inspectable scientific outputs. +2. Provide `row_id_column` when the plugin can produce stable ids. +3. Provide `coordinate_space_2d`, `coordinate_space_3d`, and `time_column` when the data supports linked 2D/3D inspection. +4. Use `layer_id` plus `HostDatasetDescriptor.display` for visibility and styling defaults. +5. Keep intentionally shared dataset/view ids byte-for-byte identical across providers. +6. Use overlays only for supplemental 2D annotation or hit-testing, not as the primary data contract. +7. When one stage needs multiple logical layers, publish separate datasets/layer ids instead of keying style by plugin name. +8. It is acceptable for multiple rows to share the same stable row id when the intended interaction is "select the whole cluster" rather than "select one raw sample". +9. Stable row keys are dataset-scoped in the current host, so matching row ids across different datasets do not create cross-dataset selection on their own. + +## Consequences + +- the host can keep selection, styling, and filtering generic +- candidate-finding can expose accepted and rejected raw events as separate investigation layers +- candidate centroid overlays can select whole raw-event clusters by reusing `cluster_id` as the stable row key for accepted events +- fitting and post-processing can safely reuse the same current-localization ids without breaking host linking +- fitting can expose rejected fits as a first-class investigation dataset instead of hiding them behind aggregate counters +- plugins carry a little more schema metadata, but avoid plugin-specific host hooks diff --git a/docs/architecture.md b/docs/architecture.md index 2e882ba..c67c61f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,6 +82,7 @@ Plugins declare host-rendered datasets and views through: The host owns: - analysis-panel rendering +- linked 2D/3D investigation state - standalone windows - dataset caching - exports @@ -89,8 +90,19 @@ The host owns: This repository currently uses that mechanism for: -- reconstruction table and density windows -- the shared EVE compact localization panel that can be provided by fitting or post-processing +- reconstruction table, density, and 3D localization inspection +- candidate-stage accepted/rejected raw-event layers for live tuning +- the shared EVE current-localization datasets that can be provided by fitting or post-processing + +For investigation-linked table datasets, the important host-consumed metadata is: + +- stable row ids via `row_id_column` +- 2D and 3D coordinates +- optional time columns +- layer ids and semantic labels +- dataset display metadata for title, default visibility, color, marker shape, and size + +Overlays remain useful for supplemental 2D annotations, but the host now treats structured datasets as the primary linking surface. ## Tradeoffs diff --git a/docs/features/README.md b/docs/features/README.md index b39d965..1b7e22b 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -4,8 +4,14 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs +- [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. +- [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. +- [Investigation Workspace Alignment](./investigation-workspace-alignment.md) — in-tree plugins updated for stable ids, linked 2D/3D/table datasets, and candidate-stage accepted/rejected event inspection. - [Plugin Runtime Migration Notes](./plugin-api-v0-2.md) — historical runtime-migration brief, updated with the current interface additions that matter to this repo. - [Plugin Host Views](./plugin-host-views.md) — generic host-rendered datasets, cache generations, and shared view ids. +- [TableV1 Declarative Metadata](./tablev1-declarative-metadata.md) — plugin-side adoption of row provenance, display formats, and cross-dataset relations for trustworthy table rendering. +- [Clickable 2D Overlays via Marker `source_row`](./clickable-overlays-source-row.md) — plugin-api ABI 4 `source_dataset_id`/`source_row_id` plumbing and failed-fit click-to-select loop. +- [Action Requests And Single-Cluster Refit](./action-requests-and-refit.md) — plugin-declared host actions, eveSMLM refit/commit/discard flow on the `augur.evesmlm.refit_preview` dataset. - [Reconstruction Workflow](./reconstruction.md) — accumulated localization tables rendered and exported by the host. - [eveSMLM Pipeline](./evesmlm.md) — candidate finding, fitting, and post-processing as three chainable plugins. diff --git a/docs/features/action-requests-and-refit.md b/docs/features/action-requests-and-refit.md new file mode 100644 index 0000000..d9be433 --- /dev/null +++ b/docs/features/action-requests-and-refit.md @@ -0,0 +1,93 @@ +# Action Requests And Single-Cluster Refit + +## Summary + +Plugins can declare host-rendered action buttons and consume the requests +the host publishes when the user triggers one. The eveSMLM fitting plugin +is the first concrete consumer: it exposes **Re-fit cluster…**, +**Commit refit**, and **Discard refit preview**. The re-fit action opens a +host-rendered modal driven by the plugin's `param_schema`, runs a +single-cluster fit with the captured parameters, and emits the result as a +separate `augur.evesmlm.refit_preview` dataset so it is visually distinct +from the main pipeline output. + +## Plugin Contract + +Refit is plumbed through the generic host action bus (see +`augur-rs/docs/features/investigation-action-requests.md`). In short: + +- Add `HostActionDescriptor` entries to `HostViewRegistry.actions` in + `host_views()`. Each descriptor declares: + - `id` — stable identifier used to route the request in `process_frame`. + - `title` — button label. + - `scope` — one of `Dataset`, `Row`, `Cluster` with the target + `dataset_id` (and `group_column` for `Cluster`). + - `param_schema: Option` — typically + `serde_json::to_value(my_settings_schema())`. Pass `None` when the + action takes no parameters. +- Read the persistent queue at `CTX_INVESTIGATION_ACTION_REQUESTS` + (`HostActionRequestQueue`). Filter by your cached + `last_consumed_action_request_id` so each request runs exactly once. +- For `Cluster` actions, expect the host to snapshot the selected rows into + `params["__augur_cluster_rows"]`. Plugins can reconstruct the selected + cluster from those rows instead of depending on the next frame to still + contain the same cluster. +- Emit side effects. Publish overlays/datasets for visual preview, or + mutate owned state for commit/discard. + +## Fitting Plugin Implementation + +- Three actions registered in `host_views()`: + - `augur.evesmlm.refit_cluster` — `Cluster` scope on + `augur.evesmlm.candidates.accepted_events` with + `group_column = "cluster_id"`. `param_schema` covers `fit_method`, + `sigma_min_nm`, `sigma_max_nm`, `max_fit_residual`. + - `augur.evesmlm.commit_refit` — `Row` scope on + `augur.evesmlm.refit_preview`, no params. + - `augur.evesmlm.discard_refit` — `Dataset` scope on + `augur.evesmlm.refit_preview`, no params. +- New persistent plugin state: + - `host_results: EveLocalizationResults` / `host_rejected_fits: Vec` — + host-visible history keyed by `cluster_id`, used for persistent tables, + 3D views, and post-commit durability across frames. + - `refit_preview_results: EveLocalizationResults` — preview rows. + - `refit_preview_replaces: Vec>` — parallel vec mapping each + preview row to the current-frame row it replaces on commit (or + `None` to append). + - `last_consumed_action_request_id: u64` — dedupe cursor. +- `process_frame` runs the normal analysis, merges the frame into the + host-visible history, drains the queue, then publishes + `CTX_EVE_LOCALIZATION_RESULTS`. Host tables/3D views therefore keep + committed rows and historical rejected fits visible across frames, while + the reconstruction-facing context publish stays frame-local. +- Preview rows render with a yellow filled-circle marker via + `add_marker_overlay`, distinct from accepted (green cross) and rejected + (red diamond). + +## Scope Resolution Details + +- **Re-fit** reconstructs the selected cluster from the host-supplied + `__augur_cluster_rows` snapshot when available, and only falls back to + the current frame's `EveCandidates` if no snapshot is present. This lets + the action work from persistent/historical selections instead of only the + latest frame. +- **Commit** matches the preview row by `row_id` parsed from the scope + payload, upserts the committed localization into the host-visible + history, drops any matching rejected-fit row for that cluster, and + updates the current frame-local results only if that cluster is still + present in the current frame. +- **Discard** clears the preview list. No other plugin state is touched. + +## Byte-Identical On Discard + +A targeted unit test +(`discard_clears_preview_without_touching_current_results`) clones +`current_results` before discard and asserts byte-identical JSON equality +after. The main pipeline output for the next frame is therefore unchanged +when a request is discarded. + +## References + +- `augur-rs/docs/adr/018-host-action-bus.md` +- `augur-rs/docs/features/investigation-action-requests.md` +- `plugins/evesmlm-fitting/src/lib.rs` diff --git a/docs/features/clickable-overlays-source-row.md b/docs/features/clickable-overlays-source-row.md new file mode 100644 index 0000000..d6ecb1e --- /dev/null +++ b/docs/features/clickable-overlays-source-row.md @@ -0,0 +1,46 @@ +# Clickable 2D Overlays via Marker `source_row` + +## Summary + +The augur-plugin-api ABI (bumped to 4) adds `source_dataset_id` and +`source_row_id` to `FfiMarkerOverlayItem`. Host-side, the viewer uses these +fields — when set — as the authoritative `StableRowKey` on click, instead of +falling back to the `(overlay.dataset_id, marker.stable_id)` pair. This lets a +plugin emit markers on one layer while pointing clicks at rows in a +*different* dataset. + +In-tree plugins now populate `source_row` explicitly: + +- `evesmlm-fitting` — accepted-localization crosses point at + `current_localizations`; rejected-fit diamonds point at `rejected_fits`. +- `evesmlm-postproc` — drift-corrected localization crosses point at + `current_localizations`. +- `evesmlm-candidates` — cluster centroid markers leave `source_row` empty + pending a cluster-addressable dataset (future work). + +## Effect on the EVE Failed-Fit Loop + +Combined with the Stage-2 `rejection_reason` headline and row provenance on +`rejected_fits`, clicking a red diamond in the 2D viewer now: + +1. selects the backing row in the rejected-fit `TableWindow`; +2. shows `rejection_reason` as the summary card heading; +3. auto-seeks the replay transport to the fit's anchor timestamp; +4. keeps the diamond visible while scrubbing inside the fit's declared span. + +No per-frame result cache is involved — the host filters declared rows by +`[span_start_us, span_end_us]` against the current frame window. + +## Code References + +| Path | Role | +| --- | --- | +| `plugins/evesmlm-fitting/src/lib.rs` | Populates `source_row` on accepted crosses and rejected diamonds | +| `plugins/evesmlm-postproc/src/lib.rs` | Populates `source_row` on drift-corrected localization crosses | +| `plugins/evesmlm-candidates/src/lib.rs` | Pending: cluster-addressable dataset for centroid markers | + +## Related + +- [TableV1 Declarative Metadata](./tablev1-declarative-metadata.md) +- [Investigation Workspace Alignment](./investigation-workspace-alignment.md) +- [eveSMLM Pipeline](./evesmlm.md) diff --git a/docs/features/evesmlm-temporal-diagnostics.md b/docs/features/evesmlm-temporal-diagnostics.md new file mode 100644 index 0000000..0efab1a --- /dev/null +++ b/docs/features/evesmlm-temporal-diagnostics.md @@ -0,0 +1,65 @@ +# EVE Temporal Diagnostics + +## Summary + +This feature extends the in-tree eveSMLM pipeline with better live diagnostics for candidate tuning and fit rejection analysis. + +The change adds: + +- temporal candidate clustering over retained event history +- provisional versus complete cluster tracking +- cluster-boundary overlays with clickable centroid markers +- rejected-fit investigation datasets and overlays + +## Candidate Finding + +`EVE Candidate Finding` can now request retained event history from the host and cluster over a configurable temporal lookback window instead of only the current preview frame. + +Tracked clusters keep a stable `cluster_id` while they are visible. A cluster is only published downstream once it has stopped growing for the configured number of stable frames. Until then it remains provisional. + +The candidate overlay now includes: + +- 2-sigma eigenfeature ellipses for DBSCAN and eigenfeature modes +- bounding boxes for frame-based mode +- clickable centroid markers linked to the accepted-events investigation dataset + +Accepted candidate-event rows now intentionally use `cluster_id` as the row-id column so one centroid click can select all raw events that belong to that cluster across the host table and 3D inspection views. + +This is intentionally scoped to the accepted candidate-events dataset. AugurRS still keys selection by `(dataset_id, row_id)`, so matching `cluster_id` values do not create automatic cross-dataset linking into rejected fits or other datasets. + +## Candidate Fitting + +`EVE Candidate Fitting` now records rejected fits with structured rejection reasons instead of only counting them. + +Rejected fits are exposed as a separate host dataset: + +- dataset id: `augur.evesmlm.rejected_fits` +- layer id: `augur.layer.evesmlm.rejected_fits` +- compact/table views for row-wise inspection +- linked 3D view: `augur.evesmlm.rejected_fits.scatter3d` + +Each rejected row carries: + +- stable `row_id` +- source `cluster_id` +- position and timestamp +- sigma values when available +- fit residual +- event count and polarity balance +- rejection reason + +The fitting status output now reports the rejection breakdown across fit failures, sigma-bound rejections, and residual-bound rejections. + +## Investigation Contracts + +This feature keeps the existing host-owned investigation model intact and extends it with two important conventions: + +1. Candidate centroid overlays link into the accepted raw-event dataset by reusing `cluster_id` as the stable row key. +2. Rejected fits are exposed as a first-class structured dataset instead of being implicit in a status count. + +## Verification + +```bash +cargo test -p augur-plugin-evesmlm-candidates +cargo test -p augur-plugin-evesmlm-fitting +``` diff --git a/docs/features/evesmlm.md b/docs/features/evesmlm.md index bdb2813..8ef6cb6 100644 --- a/docs/features/evesmlm.md +++ b/docs/features/evesmlm.md @@ -4,23 +4,38 @@ The eveSMLM pipeline is implemented as three focused plugins so each stage can b ## Stages -1. **EVE Candidate Finding** (`RawEvents`) clusters raw `CdEvent` samples into emitter candidates and publishes `EveCandidates`. -2. **EVE Candidate Fitting** (`DerivedData`) converts each candidate into one or more sub-pixel localization estimates, republishes `EveLocalizationResults` and `LocalizationResults`, and exposes the compact host-view dataset `augur.evesmlm.current_localizations`. -3. **EVE Post-Processing** (`DerivedData`) filters, drift-corrects, and evaluates the fitted localizations, then republishes the same host-view dataset id and view id with the same schema. +1. **EVE Candidate Finding** (`RawEvents`) clusters raw `CdEvent` samples into emitter candidates, can aggregate over retained event history, publishes only stable completed `EveCandidates`, and exposes accepted/rejected raw-event investigation layers plus boundary overlays. +2. **EVE Candidate Fitting** (`DerivedData`) converts each completed candidate into one or more sub-pixel localization estimates, republishes `EveLocalizationResults` and `LocalizationResults`, and exposes both the shared host-view dataset `augur.evesmlm.current_localizations` and the rejected-fit dataset `augur.evesmlm.rejected_fits`. +3. **EVE Post-Processing** (`DerivedData`) filters, drift-corrects, and evaluates the fitted localizations, then republishes the same host-view dataset id and view ids with the same schema and metadata. ## Why Three Plugins - Keeps raw-event grouping separate from numerical fitting, so candidate quality can be inspected directly. +- Lets researchers compare accepted and rejected candidate-stage raw events while tuning clustering thresholds. - Lets researchers compare fitting methods on a fixed candidate set. - Allows post-processing to be toggled or replaced without touching candidate generation. - Preserves compatibility with existing downstream plugins through `LocalizationResults`. ## Host View Resolution +- `EVE Candidate Finding` publishes two investigation datasets for the current analysis window: + - accepted candidate events + - rejected candidate events +- the accepted candidate-events dataset now keys rows by `cluster_id` so centroid overlays can select every event in a cluster at once. +- both candidate datasets now register host tables as well as 3D views, so the investigation workflow has visible table targets for selection and inspection. +- both candidate datasets include timestamps, 2D coordinates, and 3D scatter metadata so the host can color them separately in linked 2D/3D inspection. +- candidate host-view titles stay short (`Accepted Events`, `Rejected Events`) because the host renders + table/window chips in narrow plugin cards; the full dataset ids remain stable. +- candidate table display metadata marks concise `X`, `Y`, `Time`, `Polarity`, and `Cluster` + labels, with accepted events using `Cluster` as the compact-card headline. +- fitting also publishes a rejected-fit investigation dataset and 3D view so fit failures and threshold rejections can be inspected alongside accepted localizations. +- cross-dataset linking is still host-limited: matching `cluster_id` values do not automatically link candidate events to rejected fits because AugurRS selections are scoped by dataset id. - The compact EVE localization panel is declared by both fitting and post-processing. +- the 3D current-localizations view is also declared by both fitting and post-processing - The host resolves duplicate ids in plugin execution order. - When **EVE Post-Processing** is enabled, it becomes the active provider for the panel view. - When post-processing is disabled, the panel falls back automatically to **EVE Candidate Fitting**. +- fitting and post-processing must therefore keep the shared current-localization dataset/view descriptors identical ## Calibration Note @@ -30,7 +45,7 @@ The fitting and post-processing stages now use that host `nm_per_pixel` value au ## Data Flow -`CdEvent` stream -> `EveCandidates` -> `EveLocalizationResults` -> filtered / corrected `EveLocalizationResults` +`CdEvent` stream -> tracked / completed `EveCandidates` -> `EveLocalizationResults` (+ rejected-fit dataset) -> filtered / corrected `EveLocalizationResults` ## Installation diff --git a/docs/features/investigation-workspace-alignment.md b/docs/features/investigation-workspace-alignment.md new file mode 100644 index 0000000..f6d9415 --- /dev/null +++ b/docs/features/investigation-workspace-alignment.md @@ -0,0 +1,79 @@ +# Investigation Workspace Alignment + +## Summary + +This pass aligns the in-tree plugins in `augur-plugins` with the host-owned investigation workspace now implemented in `augur-rs`. + +The goal is not plugin-specific UI. The goal is to expose better generic data contracts so the host can link: + +- 2D preview points +- 3D inspection layers +- host-rendered tables + +## What Changed + +- `evesmlm-candidates` now publishes two generic raw-event investigation datasets: + - accepted candidate events + - rejected candidate events +- those candidate datasets carry: + - stable row ids + - `timestamp_us` + - 2D coordinates + - 3D coordinates using time as the `z` axis + - layer/display metadata for distinct accepted vs rejected styling +- accepted candidate rows can now intentionally share a `cluster_id` row key so one centroid overlay can select every event in that cluster +- candidate datasets must register table views as well as 3D views when the workflow expects row-wise inspection and linked selection +- `evesmlm-fitting` and `evesmlm-postproc` now keep the shared `augur.evesmlm.current_localizations` contract aligned with: + - stable row ids + - `timestamp_us` + - 2D and 3D coordinate metadata + - shared layer/display metadata + - linked marker overlays carrying stable ids +- `evesmlm-fitting` also publishes `augur.evesmlm.rejected_fits` for rejected candidates with timestamps, positions, metrics, and rejection reasons +- matching ids across different datasets still do not link automatically because the host selection model keys rows by dataset id plus stable row id +- `reconstruction` now exposes the accumulated localization dataset as a fuller investigation dataset with: + - stable row ids + - `timestamp_us` + - 3D scatter metadata + - layer/display metadata +- repo-local docs and the template guidance now describe stable ids, dataset/layer metadata, and overlays as supplemental rather than primary integration surfaces + +## Important Contracts + +### Candidate Event Layers + +The candidate-finding stage now surfaces accepted and rejected raw events from the active analysis window as separate host datasets instead of hiding that distinction inside plugin-local logic or centroid-only overlays. + +That makes it possible to tune candidate parameters while seeing: + +- which events survived into clusters +- which events were rejected +- how those two groups distribute over time in the 3D view + +### Shared EVE Current Localizations + +`evesmlm-fitting` and `evesmlm-postproc` intentionally reuse the same dataset id and view ids for current localizations. + +To keep host-side linking trustworthy, those reused descriptors must stay identical across both providers: + +- same schema +- same row-id column +- same coordinate/time metadata +- same layer metadata +- same view descriptors + +The later enabled provider can then replace the dataset payload without breaking selection, styling, or view resolution. + +### Reconstruction + +The reconstruction plugin remains generic. It still publishes one accumulated dataset as the source of truth, but that dataset now participates in the linked investigation model instead of acting only as a density-view backing store. + +## Verification + +```bash +cargo check -p augur-plugin-evesmlm-candidates +cargo test -p augur-plugin-evesmlm-candidates +cargo test -p augur-plugin-evesmlm-fitting +cargo test -p augur-plugin-evesmlm-postproc +cargo test -p augur-plugin-reconstruction +``` diff --git a/docs/features/plugin-host-views.md b/docs/features/plugin-host-views.md index 0670378..12bd335 100644 --- a/docs/features/plugin-host-views.md +++ b/docs/features/plugin-host-views.md @@ -11,19 +11,28 @@ That keeps scientific state in the plugin while letting the host own rendering, ## What Plugins Can Declare - datasets with stable ids and explicit schema metadata +- stable row ids, time columns, and 2D/3D coordinate metadata for linked investigation - analysis-panel views rendered by the host - standalone windows rendered by the host - multiple views backed by the same dataset +- layer/display metadata for host-owned visibility and styling defaults +- supplemental marker overlays for 2D hit-testing when datasets alone are not enough - optional generation counters for cache invalidation ## Current In-Tree Usage -- `Localization Reconstruction` publishes `augur.localization.accumulated` once and lets the host render both: +- `Localization Reconstruction` publishes `augur.localization.accumulated` once and lets the host render: - a `Localization Table` window - a `Reconstruction` density window -- `EVE Candidate Fitting` and `EVE Post-Processing` both publish `augur.evesmlm.current_localizations` with the same schema and the same compact panel view id + - a `Localization Cloud` 3D view +- `EVE Candidate Finding` publishes accepted and rejected raw-event datasets as separate investigation layers +- `EVE Candidate Fitting` and `EVE Post-Processing` both publish `augur.evesmlm.current_localizations` with the same schema and the same view ids -Because the host resolves duplicate ids in plugin execution order, `EVE Post-Processing` becomes the active provider whenever it is enabled; otherwise the compact table falls back to `EVE Candidate Fitting`. +Because the host resolves duplicate ids in plugin execution order, `EVE Post-Processing` becomes the active provider whenever it is enabled; otherwise the shared current-localizations dataset falls back to `EVE Candidate Fitting`. + +`Scatter3dFromTable` descriptors are consumed by AugurRS as main investigation 3D scene layers. +Plugins should still declare them with stable ids and coordinate metadata, but should not rely on +them appearing as separate dock/window chips. ## Why The Split Matters diff --git a/docs/features/plugin-install-reload.md b/docs/features/plugin-install-reload.md new file mode 100644 index 0000000..5399755 --- /dev/null +++ b/docs/features/plugin-install-reload.md @@ -0,0 +1,40 @@ +# Plugin Install And Reload + +## Goal + +Keep locally installed runtime plugins reloadable on macOS even when they are built from an in-flight sibling `augur-rs` checkout. + +## Problem + +Cargo's macOS `cdylib` outputs keep an absolute `LC_ID_DYLIB` that points back into the build tree, for example: + +```text +/path/to/augur-plugins/target/release/deps/libaugur_plugin_localization.dylib +``` + +That identity is harmless when the library stays in `target/`, but it becomes a footgun once the plugin is copied into `~/.augur/plugins//`. The host scans the installed copy, yet dyld can still treat the plugin as the build-tree image identity during later loads or reloads. + +In practice that makes plugin updates look stale: the Plugin Manager can keep reporting an older ABI or older code path even though the copied file in `~/.augur/plugins/` was rebuilt. + +## Repo-Level Fix + +- `scripts/install-built-plugins.sh` still copies each built runtime plugin into the standard `~/.augur/plugins//` layout. +- On macOS, the script now rewrites the copied library's `LC_ID_DYLIB` to `@loader_path/` with `install_name_tool`. +- That keeps the installed artifact self-identified by its installed location instead of Cargo's build-path identity, which makes rescans/reloads behave like the user expects. + +## Authoring Guidance + +- Prefer `./scripts/install-built-plugins.sh --profile release` over manual `cp` steps when installing local plugins on macOS. +- If you do copy a plugin by hand on macOS, rewrite the installed dylib id after copying: + +```bash +install_name_tool -id "@loader_path/libaugur_plugin_my_plugin.dylib" \ + ~/.augur/plugins/my-plugin/libaugur_plugin_my_plugin.dylib +``` + +- After an ABI bump in `augur-plugin-api`, rebuild the plugin and replace the installed runtime library before using **Scan for New Plugins** or **Reload** in `augur-gui`. + +## Verification + +- The installed runtime libraries continue to hash-match the built release artifacts apart from the macOS dylib id rewrite. +- `otool -D ~/.augur/plugins//libaugur_plugin_.dylib` now reports `@loader_path/...` instead of an absolute path into `target/release/deps/`. diff --git a/docs/features/reconstruction.md b/docs/features/reconstruction.md index 3320741..f85ae59 100644 --- a/docs/features/reconstruction.md +++ b/docs/features/reconstruction.md @@ -5,15 +5,26 @@ The reconstruction workflow publishes one accumulated host-view dataset instead ## Components 1. **Localization Reconstruction** (`DerivedData`) reads `LocalizationResults` from `HostContext` and stores a capped nanometer-space accumulation table. -2. **`host_views()`** declares one dataset, `augur.localization.accumulated`, plus two host-rendered window views: +2. **`host_views()`** declares one dataset, `augur.localization.accumulated`, plus host-rendered views for: - `Localization Table` - `Reconstruction` + - `Localization Cloud` 3. **`host_view_dataset()`** serves one columnar `TableV1` snapshot that both windows consume. ## Source Of Truth - the reconstruction plugin owns the only accumulated localization state -- the full table window and density reconstruction window read the same dataset id +- the full table window, density reconstruction window, and 3D scatter inspection all read the same dataset id + +## Investigation Metadata + +The accumulated localization dataset now participates directly in the host investigation workspace through: + +- stable row ids via `id` +- `timestamp_us` as the shared time column +- 2D nanometer coordinates for linked preview/table filtering +- 3D scatter coordinates using `timestamp_us` on the `z` axis +- layer/display metadata for default visibility and styling ## Resource Use @@ -28,7 +39,7 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global ## Data Flow -`LocalizationResults` -> `augur.localization.accumulated` -> host table window / host density window +`LocalizationResults` -> `augur.localization.accumulated` -> host table window / density window / 3D localization cloud ## Installation diff --git a/docs/features/tablev1-declarative-metadata.md b/docs/features/tablev1-declarative-metadata.md new file mode 100644 index 0000000..ae24fe0 --- /dev/null +++ b/docs/features/tablev1-declarative-metadata.md @@ -0,0 +1,59 @@ +# TableV1 Declarative Metadata For Plugins + +## Summary + +Plugins that expose `HostDatasetKind::TableV1` now describe row provenance, cross-dataset +relations, and per-column display formatting declaratively. The host consumes these +descriptors to render timestamps as `mm:ss.uuu`, size columns sensibly, drive summary cards, +auto-seek replay to the anchor timestamp of a selected row, and resolve derived-row selections +back to contributing raw events for 3D emphasis. + +This replaces implicit conventions (where the host guessed from type or column name) with +explicit, serializable metadata carried on `TableSchema` and `HostDatasetDescriptor`. + +## What Plugins Populate + +On `TableSchema`: + +- `provenance: Some(TableRowProvenance { anchor_time_column, span_start_column, span_end_column })` + — typically `anchor_time_column: Some("timestamp_us")`. Spans are used by the host for + span-based visibility and anchor fallback, so `span_start_column` / `span_end_column` should + describe the real contributing interval rather than repeating the anchor timestamp. +- `column_display: Vec` — one entry per column you want formatted: + - timestamp columns → `TableColumnDisplayFormat::TimestampMicros` + - positions, widths, residuals → `FixedPrecision { digits: N }` + - `row_id` columns → `Identifier` with `hidden: true` + - enum-like columns (methods, reasons) → `Category` (promote with `headline: true` in + failure-result schemas to make the reason the summary-card heading) + - Width priority: `High` (~160px) for labels and text; `Medium` (~100px) for numeric; + `Low` (~60px) for compact identifiers. + +On `HostDatasetDescriptor`: + +- `relations: Vec` — + declare joins from this dataset's row to another dataset. Example: candidate-event rows + relate to localizations via `cluster_id`. The host can follow these joins transitively to map a + selected derived row back to raw accepted-event identities. + +All new fields are additive with serde defaults; omitting them keeps the prior behavior. + +## Implemented Datasets + +- `evesmlm-fitting`: `augur.evesmlm.current_localizations`, `augur.evesmlm.rejected_fits` — full + provenance with real `span_start_us` / `span_end_us`, per-column formatting, cluster relations + back to accepted candidate events, and `rejection_reason` marked `headline: true` for rejected + fits. +- `evesmlm-candidates`: accepted/rejected candidate events — provenance on `timestamp_us`, + relation to `current_localizations` via `cluster_id` on accepted events. + +## Descriptor Parity + +`evesmlm-postproc` re-exports the `current_localizations` registry builder from +`evesmlm-fitting`, so the descriptor is structurally identical by construction. A parity test +in `plugins/evesmlm-postproc/src/lib.rs` serializes both registries to JSON and asserts +equality to catch accidental divergence. + +## Related Host Behavior + +See the companion host feature brief: [Investigation Table Trustworthiness](https://github.com/muthmann/augur-rs/blob/main/docs/features/investigation-table-trustworthiness.md) +and [ADR 017](https://github.com/muthmann/augur-rs/blob/main/docs/adr/017-declarative-tablev1-metadata.md). diff --git a/docs/installing-plugins.md b/docs/installing-plugins.md index ba8797b..45a79e9 100644 --- a/docs/installing-plugins.md +++ b/docs/installing-plugins.md @@ -47,6 +47,14 @@ cp target/release/libaugur_plugin_localization.dylib ~/.augur/plugins/localizati Install each plugin into its own directory under `~/.augur/plugins//`. +On macOS, a plain `cp` keeps Cargo's build-path dylib identity in the copied file. Rewrite the +installed copy so reloads do not keep resolving back to the build tree: + +```bash +install_name_tool -id "@loader_path/libaugur_plugin_localization.dylib" \ + ~/.augur/plugins/localization/libaugur_plugin_localization.dylib +``` + ## Install All Built Plugins ```bash @@ -54,6 +62,8 @@ Install each plugin into its own directory under `~/.augur/plugins//`. ``` This copies every plugin that already has a built runtime library in `target/release/`. +On macOS it also rewrites each installed dylib id to `@loader_path/` so Plugin Manager +reloads do not stay pinned to Cargo's original build-path identity. ## Load Or Reload In The GUI @@ -85,6 +95,16 @@ You copied a source directory instead of the built library. Build the plugin and The library was built against an older plugin interface or does not export the runtime vtable. Port it to `augur-plugin-api::Plugin` and export it with `export_plugin!`. +### “plugin ABI mismatch” + +The installed runtime library is stale relative to the host ABI. + +1. Rebuild the plugin against the current sibling `augur-rs` checkout. +2. Replace the installed runtime library in `~/.augur/plugins//`. +3. On macOS, prefer `./scripts/install-built-plugins.sh --profile release` or rewrite the copied dylib id with `install_name_tool -id "@loader_path/" ...`. + +If you overwrote a plugin while `augur-gui` was already running, restart the host once after the ABI bump to clear any previously loaded image from the process. + ### The plugin loads but host-owned settings are missing `GlobalSettings` are published through `augur.global_settings` by newer hosts. If a plugin tolerates `None` there, verify that the installed plugin and the `augur-gui` build come from compatible `augur-rs` / `augur-plugins` revisions. diff --git a/docs/plugin-api.md b/docs/plugin-api.md index f1138d6..c5c884a 100644 --- a/docs/plugin-api.md +++ b/docs/plugin-api.md @@ -1,10 +1,11 @@ # Runtime Plugin API -This repository now follows the runtime-only plugin surface documented in `augur-rs`. +This repository follows the runtime-only plugin surface documented in `augur-rs`. Use the upstream guide as the canonical contract: - [`augur-rs/docs/features/plugin-authoring-guide.md`](https://github.com/muthmann/augur-rs/blob/main/docs/features/plugin-authoring-guide.md) +- [`augur-rs/docs/features/investigation-workspace.md`](https://github.com/muthmann/augur-rs/blob/main/docs/features/investigation-workspace.md) This page summarizes the parts authors working in `augur-plugins` touch most often. @@ -25,37 +26,6 @@ This page summarizes the parts authors working in `augur-plugins` touch most oft - `Series1dV1` - `CTX_GLOBAL_SETTINGS` -## Minimal Plugin - -```rust -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostContext, HostOutput, Plugin, PluginFrame, -}; - -#[derive(Default)] -struct MyPlugin { - enabled: bool, -} - -impl Plugin for MyPlugin { - fn name(&self) -> &'static str { "My Plugin" } - fn enabled(&self) -> bool { self.enabled } - fn set_enabled(&mut self, enabled: bool) { self.enabled = enabled; } - fn reset(&mut self) {} - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - _context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - } -} - -export_plugin!(MyPlugin); -``` - ## Execution Model `input_kind()` and retained history are separate concerns. @@ -94,64 +64,42 @@ context.publish("my.plugin.results", &results)?; let upstream = context.get::("my.plugin.results")?; ``` -Prefer standard shared payloads such as `CTX_LOCALIZATION_RESULTS` when they exist. The standard localization payload now lives in `augur-plugin-types`. If several plugins need the same domain-specific type, put that type in a companion crate instead of copying it into multiple plugin crates. - -Persistent helpers are still available for plugin-owned caches, but shared scientific outputs should normally stay on the per-frame context bus. +Prefer standard shared payloads such as `CTX_LOCALIZATION_RESULTS` when they exist. If several plugins need the same domain-specific type, put that type in a companion crate instead of copying it into multiple plugin crates. ## Host-Owned Global Settings -The host now publishes shared runtime settings on the normal context bus: +The host publishes shared runtime settings on the normal context bus: - key: `CTX_GLOBAL_SETTINGS` - type: `GlobalSettings` -Example: - -```rust -use augur_plugin_api::{GlobalSettings, CTX_GLOBAL_SETTINGS}; - -if let Some(globals) = context.get::(CTX_GLOBAL_SETTINGS)? { - let nm_per_pixel = globals.nm_per_pixel; - let sensor_width = globals.sensor_width; - let sensor_height = globals.sensor_height; - let acq_time_ms = globals.acq_time_ms; - let event_store_budget_bytes = globals.event_store_budget_bytes; - let _ = ( - nm_per_pixel, - sensor_width, - sensor_height, - acq_time_ms, - event_store_budget_bytes, - ); -} -``` - New plugins should prefer `GlobalSettings` over duplicating host-owned defaults such as pixel scale or sensor geometry. -Plugins must tolerate `None` when run against an older host build. - -## Dependencies - -Override `dependencies()` only when the plugin truly requires a specific upstream producer by name: - -```rust -fn dependencies(&self) -> &[&'static str] { - &["EVE Candidate Finding"] -} -``` +## Linked Investigation Datasets -If the plugin can degrade gracefully when an upstream payload is absent, prefer a runtime warning over a hard dependency declaration. +The host now treats structured datasets as the primary integration surface for linked 2D, 3D, and table workflows. Overlays are supplemental. -## Settings And Status +When a table dataset should participate in linked investigation, populate as many of these additive fields as the plugin can support: -Plugins describe settings declaratively through: +- `TableSchema.coordinate_space_2d` +- `TableSchema.coordinate_space_3d` +- `TableSchema.row_id_column` +- `TableSchema.time_column` +- `TableSchema.layer_id` +- `TableSchema.semantic_label` +- `HostDatasetDescriptor.display` + - `layer_title` + - `default_visibility` + - `default_color` + - `default_marker_shape` + - `default_size` -- `settings_schema()` -- `get_setting()` -- `set_setting()` -- optional `status_entries()` +Guidelines: -Common setting kinds include `Bool`, slider/drag values, and `Enum`. The host owns rendering and persistence of the UI state. +- Use stable ids from the scientific data when possible. +- Fall back to deterministic plugin-generated ids when no natural id exists. +- Key reusable shared views by dataset id and keep descriptors byte-for-byte identical across providers that intentionally reuse the same ids. +- Prefer dataset/layer ids for styling and visibility instead of plugin-name-specific logic. ## Host Views @@ -169,6 +117,7 @@ Plugins can declare host-rendered datasets and views through `host_views()` and - `HostViewKind::TableWindow` - `HostViewKind::Density2dFromTable` - `HostViewKind::Scatter2dFromTable` +- `HostViewKind::Scatter3dFromTable` - `HostViewKind::ImageWindow` - `HostViewKind::LineSeriesWindow` @@ -178,10 +127,20 @@ Plugins can declare host-rendered datasets and views through `host_views()` and fn host_views(&self) -> HostViewRegistry { HostViewRegistry { datasets: vec![HostDatasetDescriptor { - id: "example.table".into(), - title: "Example Table".into(), + id: "example.points".into(), + title: "Example Points".into(), kind: HostDatasetKind::TableV1(TableSchema { columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, TableColumn { id: "x".into(), title: "X".into(), @@ -193,48 +152,79 @@ fn host_views(&self) -> HostViewRegistry { value_type: TableValueType::F64, }, ], - coordinate_space_2d: None, + coordinate_space_2d: Some(TableCoordinateSpace2d { + x_column: "x".into(), + y_column: "y".into(), + x_min: 0.0, + x_max: 128.0, + y_min: 0.0, + y_max: 128.0, + }), + coordinate_space_3d: Some(TableCoordinateSpace3d { + x_column: "x".into(), + y_column: "y".into(), + z_column: "timestamp_us".into(), + x_min: 0.0, + x_max: 128.0, + y_min: 0.0, + y_max: 128.0, + z_min: 0.0, + z_max: 5_000.0, + }), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some("example.layer.points".into()), + semantic_label: Some("points".into()), }), empty_message: "No rows yet.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Example points".into()), + default_visibility: Some(true), + default_color: Some([80, 200, 255, 255]), + default_marker_shape: Some(HostMarkerShape::Point), + default_size: Some(3.0), + }), }], views: vec![HostViewDescriptor { - id: "example.table.compact".into(), - title: "Current Rows".into(), - dataset_id: "example.table".into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, + id: "example.points.3d".into(), + title: "Example 3D".into(), + dataset_id: "example.points".into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x".into(), + y_column: "y".into(), + z_column: "timestamp_us".into(), + }, }], } } - -fn host_view_dataset(&self, dataset_id: &str) -> Option> { - if dataset_id != "example.table" { - return None; - } - - let dataset = TableDatasetV1::new(vec![ - TableColumnData { - column_id: "x".into(), - values: TableColumnValues::F64(vec![1.0, 2.0]), - }, - TableColumnData { - column_id: "y".into(), - values: TableColumnValues::F64(vec![3.0, 4.0]), - }, - ]).ok()?; - - serde_json::to_vec(&dataset).ok() -} - -fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - if dataset_id == "example.table" { 1 } else { 0 } -} ``` `host_view_dataset_generation()` is optional but recommended when the host should invalidate a cached snapshot only after the dataset changes. The host owns rendering, exports, caching, and window state. Plugins do not render `egui` directly. +## Marker Overlays + +Use structured datasets for the primary linked-workspace model. Use overlays when the plugin needs extra 2D annotations or hit-testing that supplements the dataset. + +Current overlay helpers: + +- `add_highlight_pixels(...)` +- `add_crosshair_markers(...)` +- `add_marker_overlay(...)` +- `add_warning(...)` + +`add_marker_overlay(...)` supports: + +- point, cross, box, ellipse, diamond, and filled-circle shapes +- per-item color and size +- optional timestamp +- optional stable id +- optional dataset id, layer id, and source label + +That makes it the right choice when a 2D preview annotation should resolve back into the same host selection model. + ## Event History `process_frame()` always receives `event_store: &EventStoreHandle<'_>`. Plugins that need only the current frame can ignore it. History-aware plugins can query: @@ -247,13 +237,14 @@ The host owns rendering, exports, caching, and window state. Plugins do not rend - `collect_events_in_range(start_us, end_us, out)` - `oldest_timestamp_us()` -## Migration From Older Plugin Code +## Migration Notes -When porting older code, replace: +When porting older code: -- `AnalysisPlugin` with `Plugin` -- typed `PluginContext` exchange with `HostContext` -- direct `egui` UI code with declarative settings/status -- special-case host rendering hooks with `host_views()` and `host_view_dataset()` -- duplicated host-owned calibration values with `GlobalSettings` -- compile-time registration with `export_plugin!` plus a built `cdylib` +- replace `AnalysisPlugin` with `Plugin` +- replace typed `PluginContext` exchange with `HostContext` +- replace direct `egui` UI code with declarative settings/status +- replace special-case host rendering hooks with `host_views()` / `host_view_dataset()` +- replace row-index-based linking assumptions with stable row ids where possible +- replace plugin-name-based styling assumptions with dataset/layer metadata +- keep overlays as supplemental annotations, not the primary data contract diff --git a/plugin-template/README.md b/plugin-template/README.md index d525db8..ae32b4c 100644 --- a/plugin-template/README.md +++ b/plugin-template/README.md @@ -22,6 +22,15 @@ If this plugin publishes results to `HostContext` for downstream consumers, desc If this plugin declares datasets or views through `host_views()`, document the dataset ids, view ids, and expected schema here. +For investigation-linked table datasets, also document: + +- which column provides stable row identity +- whether 2D coordinates are exposed +- whether 3D coordinates and time are exposed +- which layer id and display defaults the host should expect + +Prefer structured datasets for linked 2D/3D/table workflows. Use overlays as supplemental annotations rather than the only way to inspect results. + ## Dependencies List any hard upstream plugin dependencies this plugin declares through `dependencies()` (or "None"). diff --git a/plugins/evesmlm-candidates/README.md b/plugins/evesmlm-candidates/README.md index de3a9a1..89c1b47 100644 --- a/plugins/evesmlm-candidates/README.md +++ b/plugins/evesmlm-candidates/README.md @@ -20,26 +20,43 @@ Raw-event candidate discovery for eveSMLM. This plugin groups `CdEvent` samples | Polarity | `Both` | Use positive, negative, or all events | | Epsilon | `3.0` px | Neighborhood radius for DBSCAN | | Min events | `5` | Minimum cluster size | +| Lookback | `66_000` us | Retained-history window used for temporal clustering; set to `0` for single-frame behavior | +| Stable frames | `2` | Number of consecutive no-growth frames before a cluster is published | | Max spatial extent | `5.0` px | Eigenfeature upper bound on the major covariance axis | | Min isotropy | `0.2` | Eigenfeature lower bound on `lambda2 / lambda1` | | Threshold factor | `1.5` | Wavelet threshold multiplier for frame-based mode | | Fit radius | `4` px | Event gathering radius in frame-based mode | | Max candidates | `512` | Safety cap on published candidates | -| Show overlay | `true` | Highlight candidate centroids in the preview | +| Show centroids | `true` | Draw clickable centroid markers linked to accepted candidate events | +| Show boundaries | `true` | Draw 2-sigma ellipses or bounding boxes around visible clusters | +| Show provisional | `true` | Keep still-growing clusters visible in the overlay | ## Execution Phase -`RawEvents` — consumes the raw `CdEvent` stream for the current preview window. +`RawEvents` — consumes the raw `CdEvent` stream for the current preview window and can optionally gather retained events from earlier frames. ## Published Data Publishes `EveCandidates` on the context key `augur.evesmlm.candidates`, containing: -- `clusters: Vec` with raw events, per-pixel histograms, centroid, and bounds +- `clusters: Vec` with stable `cluster_id`, raw events, per-pixel histograms, centroid, bounds, and optional boundary metadata - `frame_window_start_us`, `frame_window_end_us` - `n_events_processed` - `finding_method` +It also exposes two host investigation datasets for the current analysis window: + +- accepted candidate events +- rejected candidate events + +Both datasets carry stable row ids, timestamps, 2D coordinates, and 3D scatter metadata so the host can render accepted and rejected raw events as separate layers during live parameter tuning. + +The plugin now also registers compact and windowed host tables for both datasets, so centroid selection has a visible table target inside AugurRS without requiring plugin-specific UI. + +Accepted candidate-event rows intentionally use the string form of `cluster_id` as the row-id column so one centroid click can select the whole cluster in the accepted-events table and its 3D view. + +That selection is dataset-local: it links the centroid marker to the accepted-events dataset, but it does not cross-select unrelated datasets such as rejected fits because AugurRS stable row keys include the dataset id. + ## Dependencies None. diff --git a/plugins/evesmlm-candidates/src/eigenfeature.rs b/plugins/evesmlm-candidates/src/eigenfeature.rs index e75df0c..60a9e7f 100644 --- a/plugins/evesmlm-candidates/src/eigenfeature.rs +++ b/plugins/evesmlm-candidates/src/eigenfeature.rs @@ -2,6 +2,13 @@ use nalgebra::Matrix2; use crate::EveEvent; +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ClusterEigenInfo { + pub lambda_1: f64, + pub lambda_2: f64, + pub angle_rad: f64, +} + pub fn filter_clusters( events: &[EveEvent], clusters: Vec>, @@ -13,20 +20,20 @@ pub fn filter_clusters( clusters .into_iter() .filter(|indices| { - let Some((lambda_1, lambda_2)) = cluster_eigenvalues(events, indices) else { + let Some(info) = cluster_eigen_info(events, indices) else { return false; }; - let isotropy = if lambda_1 <= 1e-9 { + let isotropy = if info.lambda_1 <= 1e-9 { 1.0 } else { - lambda_2 / lambda_1 + info.lambda_2 / info.lambda_1 }; - lambda_1 <= max_variance && isotropy >= min_isotropy + info.lambda_1 <= max_variance && isotropy >= min_isotropy }) .collect() } -pub fn cluster_eigenvalues(events: &[EveEvent], indices: &[usize]) -> Option<(f64, f64)> { +pub fn cluster_eigen_info(events: &[EveEvent], indices: &[usize]) -> Option { if indices.len() < 2 { return None; } @@ -55,7 +62,21 @@ pub fn cluster_eigenvalues(events: &[EveEvent], indices: &[usize]) -> Option<(f6 covariance /= n.max(1.0); let eigen = covariance.symmetric_eigen(); - let mut eigenvalues = [eigen.eigenvalues[0], eigen.eigenvalues[1]]; - eigenvalues.sort_by(|left, right| right.total_cmp(left)); - Some((eigenvalues[0], eigenvalues[1])) + let major_index = if eigen.eigenvalues[0] >= eigen.eigenvalues[1] { + 0 + } else { + 1 + }; + let minor_index = 1 - major_index; + let major_vector = eigen.eigenvectors.column(major_index); + + Some(ClusterEigenInfo { + lambda_1: eigen.eigenvalues[major_index], + lambda_2: eigen.eigenvalues[minor_index], + angle_rad: major_vector[1].atan2(major_vector[0]), + }) +} + +pub fn cluster_eigenvalues(events: &[EveEvent], indices: &[usize]) -> Option<(f64, f64)> { + cluster_eigen_info(events, indices).map(|info| (info.lambda_1, info.lambda_2)) } diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index 8b3d215..e0bc71e 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -8,16 +8,26 @@ pub mod dbscan; pub mod eigenfeature; pub mod types; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use augur_plugin_api::{ - export_plugin, AnalysisSeverity, EventStoreHandle, FfiCdEvent, FfiPixel, HostContext, - HostOutput, Plugin, PluginFrame, PluginInput, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, + export_plugin, AnalysisSeverity, EventStoreHandle, FfiCdEvent, FfiColorRgba, + FfiMarkerOverlayItem, FfiMarkerShape, FfiPixel, FfiString, HostContext, HostDatasetDescriptor, + HostDatasetDisplayMetadata, HostDatasetKind, HostDatasetRelation, HostMarkerShape, HostOutput, + HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, + PluginCapabilities, PluginFrame, PluginInput, PluginStateKind, SettingItem, SettingKind, + SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, + TableColumnDisplayEntry, TableColumnDisplayFormat, TableColumnDisplayMetadata, TableColumnValues, + TableColumnWidthPriority, TableCoordinateSpace2d, TableCoordinateSpace3d, TableDatasetV1, + TableRowProvenance, TableSchema, TableValueType, }; use serde_json::{json, Value}; -pub use types::{CandidateFindingMethod, EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES}; +use types::TrackedCluster; +pub use types::{ + CandidateFindingMethod, ClusterBoundary, EveCandidates, EveCluster, EveEvent, + CTX_EVE_CANDIDATES, +}; const KERNEL_G1: [f64; 5] = [1.0 / 16.0, 0.25, 3.0 / 8.0, 0.25, 1.0 / 16.0]; const KERNEL_G2: [f64; 9] = [ @@ -31,7 +41,57 @@ const KERNEL_G2: [f64; 9] = [ 0.0, 1.0 / 16.0, ]; -const OVERLAY_COLOR: [u8; 4] = [255, 210, 32, 220]; +const ACCEPTED_EVENTS_COLOR: [u8; 4] = [60, 220, 140, 255]; +const REJECTED_EVENTS_COLOR: [u8; 4] = [255, 110, 110, 235]; +const COMPLETE_BOUNDARY_COLOR: [u8; 4] = [255, 255, 255, 60]; +const PROVISIONAL_BOUNDARY_COLOR: [u8; 4] = [255, 255, 255, 28]; +const COMPLETE_MARKER_COLOR: [u8; 4] = [255, 255, 255, 180]; +const PROVISIONAL_MARKER_COLOR: [u8; 4] = [255, 255, 255, 110]; +const ACCEPTED_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; +const REJECTED_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.rejected_events"; +const ACCEPTED_EVENTS_LAYER_ID: &str = "augur.layer.evesmlm.accepted_events"; +const REJECTED_EVENTS_LAYER_ID: &str = "augur.layer.evesmlm.rejected_events"; +const ACCEPTED_EVENTS_COMPACT_VIEW_ID: &str = "augur.evesmlm.candidates.accepted_events.compact"; +const REJECTED_EVENTS_COMPACT_VIEW_ID: &str = "augur.evesmlm.candidates.rejected_events.compact"; +const ACCEPTED_EVENTS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.accepted_events.table"; +const REJECTED_EVENTS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.rejected_events.table"; +const ACCEPTED_EVENTS_3D_VIEW_ID: &str = "augur.evesmlm.candidates.accepted_events.scatter3d"; +const REJECTED_EVENTS_3D_VIEW_ID: &str = "augur.evesmlm.candidates.rejected_events.scatter3d"; +const CANDIDATE_FINDINGS_DATASET_ID: &str = "augur.evesmlm.candidates.candidate_findings"; +const CANDIDATE_FINDING_PIXELS_DATASET_ID: &str = + "augur.evesmlm.candidates.candidate_finding_pixels"; +const CANDIDATE_FINDINGS_LAYER_ID: &str = "augur.layer.evesmlm.candidate_findings"; +const CANDIDATE_FINDINGS_COMPACT_VIEW_ID: &str = + "augur.evesmlm.candidates.candidate_findings.compact"; +const CANDIDATE_FINDINGS_TABLE_VIEW_ID: &str = + "augur.evesmlm.candidates.candidate_findings.table"; +const CANDIDATE_FINDING_PIXELS_TABLE_VIEW_ID: &str = + "augur.evesmlm.candidates.candidate_finding_pixels.table"; + +#[derive(Debug, Clone)] +struct CandidateEventRow { + event_id: u64, + x_px: f64, + y_px: f64, + timestamp_us: u64, + polarity: bool, + cluster_id: String, +} + +#[derive(Debug, Clone, Default)] +struct CandidateEventDatasets { + accepted: Vec, + rejected: Vec, + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +} + +#[derive(Debug, Clone)] +struct CandidateFinding { + cluster: EveCluster, + method: CandidateFindingMethod, +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum PolarityMode { @@ -99,12 +159,16 @@ pub struct CandidateSettings { pub polarity: PolarityMode, pub epsilon_px: f64, pub min_events: usize, + pub lookback_us: u64, + pub stable_frames: usize, pub max_spatial_extent_px: f64, pub min_isotropy: f64, pub threshold_factor: f64, pub fit_radius_px: usize, pub max_candidates: usize, pub show_overlay: bool, + pub show_boundaries: bool, + pub show_provisional: bool, } impl Default for CandidateSettings { @@ -114,12 +178,16 @@ impl Default for CandidateSettings { polarity: PolarityMode::Both, epsilon_px: 3.0, min_events: 5, + lookback_us: 66_000, + stable_frames: 2, max_spatial_extent_px: 5.0, min_isotropy: 0.2, threshold_factor: 1.5, fit_radius_px: 4, max_candidates: 512, show_overlay: true, + show_boundaries: true, + show_provisional: true, } } } @@ -127,9 +195,19 @@ impl Default for CandidateSettings { pub struct EveSmlmCandidatePlugin { enabled: bool, settings: CandidateSettings, + current_event_datasets: CandidateEventDatasets, last_candidate_count: usize, + last_complete_visible_count: usize, + last_provisional_count: usize, last_event_count: usize, last_status: String, + dataset_generation: u64, + findings: Vec, + findings_generation: u64, + frame_counter: u64, + next_cluster_id: u64, + tracked_clusters: Vec, + event_buffer: Vec, } impl Default for EveSmlmCandidatePlugin { @@ -137,24 +215,106 @@ impl Default for EveSmlmCandidatePlugin { Self { enabled: false, settings: CandidateSettings::default(), + current_event_datasets: CandidateEventDatasets::default(), last_candidate_count: 0, + last_complete_visible_count: 0, + last_provisional_count: 0, last_event_count: 0, last_status: "Enable the plugin to cluster raw eveSMLM events into candidates.".into(), + dataset_generation: 0, + findings: Vec::new(), + findings_generation: 0, + frame_counter: 0, + next_cluster_id: 0, + tracked_clusters: Vec::new(), + event_buffer: Vec::new(), } } } impl EveSmlmCandidatePlugin { + fn reset_tracking_state(&mut self) { + self.frame_counter = 0; + self.next_cluster_id = 0; + self.tracked_clusters.clear(); + self.event_buffer.clear(); + self.findings.clear(); + self.findings_generation = self.findings_generation.wrapping_add(1); + } + + fn append_findings(&mut self, clusters: &[EveCluster]) { + if clusters.is_empty() { + return; + } + + let method = self.settings.finding_method; + self.findings + .extend(clusters.iter().cloned().map(|cluster| CandidateFinding { + cluster, + method, + })); + self.findings_generation = self.findings_generation.wrapping_add(1); + } + + fn collect_analysis_events( + &mut self, + frame: &PluginFrame<'_>, + event_store: &EventStoreHandle<'_>, + ) -> (Vec, u64, u64, bool) { + let mut analysis_events = std::mem::take(&mut self.event_buffer); + let mut analysis_window_start = frame.window_start_us(); + let analysis_window_end = frame.window_end_us(); + let temporal_enabled = self.settings.lookback_us > 0 && event_store.frame_count() > 0; + + analysis_events.clear(); + if temporal_enabled { + let buffered_start = analysis_window_end.saturating_sub(self.settings.lookback_us); + analysis_window_start = buffered_start.max(event_store.oldest_timestamp_us()); + event_store.collect_events_in_range( + analysis_window_start, + analysis_window_end, + &mut analysis_events, + ); + } + + if analysis_events.is_empty() { + analysis_events.extend_from_slice(frame.events()); + analysis_window_start = frame.window_start_us(); + } + + ( + analysis_events, + analysis_window_start, + analysis_window_end, + temporal_enabled, + ) + } + fn analyze_frame( &mut self, frame: &PluginFrame<'_>, raw_events: &[FfiCdEvent], + analysis_window_start_us: u64, + analysis_window_end_us: u64, + temporal_enabled: bool, output: &mut HostOutput<'_>, ) -> EveCandidates { if raw_events.is_empty() { + self.current_event_datasets = CandidateEventDatasets { + sensor_dims: Some((frame.width(), frame.height())), + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, + ..CandidateEventDatasets::default() + }; self.last_candidate_count = 0; + self.last_complete_visible_count = 0; + self.last_provisional_count = 0; self.last_event_count = 0; - self.last_status = "Raw events are unavailable for this preview frame.".into(); + self.last_status = if temporal_enabled { + "No retained raw events are available in the requested temporal lookback.".into() + } else { + "Raw events are unavailable for this preview frame.".into() + }; Self::warning( output, AnalysisSeverity::Info, @@ -171,18 +331,26 @@ impl EveSmlmCandidatePlugin { .collect(); self.last_event_count = filtered_events.len(); if filtered_events.is_empty() { + self.current_event_datasets = CandidateEventDatasets { + sensor_dims: Some((frame.width(), frame.height())), + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, + ..CandidateEventDatasets::default() + }; self.last_candidate_count = 0; + self.last_complete_visible_count = 0; + self.last_provisional_count = 0; self.last_status = "No events passed the configured polarity filter.".into(); return EveCandidates { clusters: Vec::new(), - frame_window_start_us: frame.window_start_us(), - frame_window_end_us: frame.window_end_us(), + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, n_events_processed: 0, finding_method: self.settings.finding_method, }; } - let cluster_indices = match self.settings.finding_method { + let mut cluster_indices = match self.settings.finding_method { CandidateFindingMethod::Dbscan => dbscan::cluster_event_indices( &filtered_events, self.settings.epsilon_px, @@ -207,44 +375,264 @@ impl EveSmlmCandidatePlugin { } }; - let mut clusters = clusters_from_indices(&filtered_events, cluster_indices); - clusters.sort_by_key(|cluster| std::cmp::Reverse(cluster.event_count())); - if clusters.len() > self.settings.max_candidates { - clusters.truncate(self.settings.max_candidates); + cluster_indices.sort_by_key(|indices| std::cmp::Reverse(indices.len())); + if cluster_indices.len() > self.settings.max_candidates { + cluster_indices.truncate(self.settings.max_candidates); } - self.last_candidate_count = clusters.len(); - self.last_status = format!( - "{} candidates from {} events using {}.", - self.last_candidate_count, - self.last_event_count, - self.settings.finding_method.label() + let detected_clusters = clusters_from_indices( + &filtered_events, + cluster_indices.clone(), + self.settings.finding_method, + ); + let (visible_clusters, published_clusters) = + self.update_tracked_clusters(detected_clusters, temporal_enabled); + self.append_findings(&published_clusters); + + self.current_event_datasets = build_candidate_event_datasets( + (frame.width(), frame.height()), + analysis_window_start_us, + analysis_window_end_us, + &filtered_events, + &cluster_indices, + &visible_clusters, ); - if self.settings.show_overlay && !clusters.is_empty() { - let pixels: Vec = clusters - .iter() - .map(|cluster| FfiPixel { - x: cluster.centroid_x.round().max(0.0) as u16, - y: cluster.centroid_y.round().max(0.0) as u16, - }) - .collect(); - output.add_highlight_pixels(&pixels, OVERLAY_COLOR); - } + self.last_candidate_count = published_clusters.len(); + self.last_complete_visible_count = visible_clusters + .iter() + .filter(|cluster| cluster.complete) + .count(); + self.last_provisional_count = visible_clusters + .len() + .saturating_sub(self.last_complete_visible_count); + + let window_span_us = analysis_window_end_us.saturating_sub(analysis_window_start_us); + let boundary_summary = if self.settings.show_boundaries && !visible_clusters.is_empty() { + format!( + " Showing {} for {} visible clusters.", + boundary_label(self.settings.finding_method), + visible_clusters.len() + ) + } else { + String::new() + }; + self.last_status = if temporal_enabled { + format!( + "{} published, {} complete visible, {} provisional from {} events using {} over {} us.{}", + self.last_candidate_count, + self.last_complete_visible_count, + self.last_provisional_count, + self.last_event_count, + self.settings.finding_method.label(), + window_span_us, + boundary_summary + ) + } else { + format!( + "{} published from {} events using {} in the current frame.{}", + self.last_candidate_count, + self.last_event_count, + self.settings.finding_method.label(), + boundary_summary + ) + }; + + self.render_cluster_overlay(frame, &visible_clusters, output); EveCandidates { - clusters, - frame_window_start_us: frame.window_start_us(), - frame_window_end_us: frame.window_end_us(), + clusters: published_clusters, + frame_window_start_us: analysis_window_start_us, + frame_window_end_us: analysis_window_end_us, n_events_processed: filtered_events.len(), finding_method: self.settings.finding_method, } } + fn update_tracked_clusters( + &mut self, + mut detected_clusters: Vec, + temporal_enabled: bool, + ) -> (Vec, Vec) { + self.frame_counter = self.frame_counter.wrapping_add(1); + let stable_frames = self.settings.stable_frames.max(1); + let retention_frames = stable_frames.saturating_mul(2).max(1); + let matching_radius = self.settings.epsilon_px.max(0.5); + + let mut candidate_pairs = Vec::new(); + for (detected_index, cluster) in detected_clusters.iter().enumerate() { + for (tracked_index, tracked) in self.tracked_clusters.iter().enumerate() { + let dx = cluster.centroid_x - tracked.centroid_x; + let dy = cluster.centroid_y - tracked.centroid_y; + let distance = (dx * dx + dy * dy).sqrt(); + if distance <= matching_radius { + candidate_pairs.push((distance, detected_index, tracked_index)); + } + } + } + candidate_pairs.sort_by(|left, right| left.0.total_cmp(&right.0)); + + let mut detected_to_tracked = vec![None; detected_clusters.len()]; + let mut tracked_taken = vec![false; self.tracked_clusters.len()]; + for (_, detected_index, tracked_index) in candidate_pairs { + if detected_to_tracked[detected_index].is_none() && !tracked_taken[tracked_index] { + detected_to_tracked[detected_index] = Some(tracked_index); + tracked_taken[tracked_index] = true; + } + } + + for (detected_index, cluster) in detected_clusters.iter_mut().enumerate() { + if let Some(tracked_index) = detected_to_tracked[detected_index] { + let tracked = &mut self.tracked_clusters[tracked_index]; + let current_count = cluster.event_count(); + tracked.centroid_x = cluster.centroid_x; + tracked.centroid_y = cluster.centroid_y; + tracked.last_seen_frame = self.frame_counter; + + if current_count > tracked.event_count { + tracked.event_count = current_count; + tracked.last_grown_frame = self.frame_counter; + tracked.frames_since_growth = 0; + tracked.cluster = cluster.clone(); + if temporal_enabled { + tracked.complete = false; + } + } else { + tracked.frames_since_growth = tracked.frames_since_growth.saturating_add(1); + if current_count == tracked.event_count { + tracked.cluster = cluster.clone(); + } + } + + if !temporal_enabled || tracked.frames_since_growth >= stable_frames { + tracked.complete = true; + } + + tracked.cluster.cluster_id = tracked.id; + tracked.cluster.complete = tracked.complete; + cluster.cluster_id = tracked.id; + cluster.complete = tracked.complete; + } else { + let cluster_id = self.next_cluster_id; + self.next_cluster_id = self.next_cluster_id.wrapping_add(1); + cluster.cluster_id = cluster_id; + cluster.complete = !temporal_enabled; + self.tracked_clusters.push(TrackedCluster { + id: cluster_id, + centroid_x: cluster.centroid_x, + centroid_y: cluster.centroid_y, + event_count: cluster.event_count(), + last_seen_frame: self.frame_counter, + last_grown_frame: self.frame_counter, + frames_since_growth: 0, + complete: cluster.complete, + emitted: false, + cluster: cluster.clone(), + }); + } + } + + for tracked in &mut self.tracked_clusters { + if tracked.last_seen_frame != self.frame_counter { + tracked.frames_since_growth = tracked.frames_since_growth.saturating_add(1); + if temporal_enabled && tracked.frames_since_growth >= stable_frames { + tracked.complete = true; + } + } + if !temporal_enabled { + tracked.complete = true; + } + tracked.cluster.cluster_id = tracked.id; + tracked.cluster.complete = tracked.complete; + } + + let mut published_clusters = Vec::new(); + for tracked in &mut self.tracked_clusters { + if tracked.complete && !tracked.emitted { + tracked.emitted = true; + let mut cluster = tracked.cluster.clone(); + cluster.cluster_id = tracked.id; + cluster.complete = true; + published_clusters.push(cluster); + } + } + + self.tracked_clusters.retain(|tracked| { + self.frame_counter.saturating_sub(tracked.last_seen_frame) as usize <= retention_frames + }); + + (detected_clusters, published_clusters) + } + + fn render_cluster_overlay( + &self, + frame: &PluginFrame<'_>, + visible_clusters: &[EveCluster], + output: &mut HostOutput<'_>, + ) { + let overlay_clusters: Vec<&EveCluster> = visible_clusters + .iter() + .filter(|cluster| cluster.complete || self.settings.show_provisional) + .collect(); + + if self.settings.show_boundaries && !overlay_clusters.is_empty() { + let (complete_pixels, provisional_pixels) = + boundary_pixels(&overlay_clusters, frame.width(), frame.height()); + if !complete_pixels.is_empty() { + output.add_highlight_pixels(&complete_pixels, COMPLETE_BOUNDARY_COLOR); + } + if !provisional_pixels.is_empty() { + output.add_highlight_pixels(&provisional_pixels, PROVISIONAL_BOUNDARY_COLOR); + } + } + + if self.settings.show_overlay && !overlay_clusters.is_empty() { + let stable_ids: Vec = overlay_clusters + .iter() + .map(|cluster| cluster.cluster_id.to_string()) + .collect(); + let markers: Vec = overlay_clusters + .iter() + .zip(stable_ids.iter()) + .map(|(cluster, stable_id)| FfiMarkerOverlayItem { + x: cluster.centroid_x as f32, + y: cluster.centroid_y as f32, + shape: FfiMarkerShape::FilledCircle, + size: 4.0, + color: FfiColorRgba::from_rgba(if cluster.complete { + COMPLETE_MARKER_COLOR + } else { + PROVISIONAL_MARKER_COLOR + }), + timestamp_us: cluster + .events + .last() + .map(|event| event.timestamp) + .unwrap_or(frame.window_end_us()), + has_timestamp: !cluster.events.is_empty(), + stable_id: stable_id.as_str().into(), + source_dataset_id: FfiString::empty(), + source_row_id: FfiString::empty(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(ACCEPTED_EVENTS_DATASET_ID), + Some(ACCEPTED_EVENTS_LAYER_ID), + Some(self.name()), + ); + } + } + pub fn reset(&mut self) { + self.reset_tracking_state(); + self.current_event_datasets = CandidateEventDatasets::default(); self.last_candidate_count = 0; + self.last_complete_visible_count = 0; + self.last_provisional_count = 0; self.last_event_count = 0; self.last_status = "Waiting for the next preview frame.".into(); + self.dataset_generation = self.dataset_generation.wrapping_add(1); } fn parse_usize(value: Value) -> Option { @@ -289,9 +677,20 @@ impl Plugin for EveSmlmCandidatePlugin { frame: &PluginFrame<'_>, output: &mut HostOutput<'_>, context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, + event_store: &EventStoreHandle<'_>, ) { - let candidates = self.analyze_frame(frame, frame.events(), output); + let (analysis_events, analysis_window_start_us, analysis_window_end_us, temporal_enabled) = + self.collect_analysis_events(frame, event_store); + let candidates = self.analyze_frame( + frame, + &analysis_events, + analysis_window_start_us, + analysis_window_end_us, + temporal_enabled, + output, + ); + self.event_buffer = analysis_events; + self.dataset_generation = self.dataset_generation.wrapping_add(1); if let Err(err) = context.publish(CTX_EVE_CANDIDATES, &candidates) { Self::warning( output, @@ -301,6 +700,12 @@ impl Plugin for EveSmlmCandidatePlugin { } } + fn capabilities(&self) -> PluginCapabilities { + PluginCapabilities { + retained_event_history: self.settings.lookback_us > 0, + } + } + fn settings_schema(&self) -> SettingsSchema { SettingsSchema { sections: vec![ @@ -373,10 +778,59 @@ impl Plugin for EveSmlmCandidatePlugin { }, ], }, + SettingsSection { + label: "Temporal aggregation".into(), + description: Some( + "Optionally cluster across retained event history and only publish clusters once they stop growing." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "lookback_us".into(), + label: "Lookback".into(), + tooltip: Some( + "How far back in retained event history to gather events before clustering. Set to 0 for single-frame behavior." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: 500_000, + default: i64::try_from(self.settings.lookback_us).unwrap_or(66_000), + suffix: Some(" us".into()), + }, + }, + SettingItem { + key: "stable_frames".into(), + label: "Stable frames".into(), + tooltip: Some( + "How many consecutive frames without cluster growth are required before a cluster is published to fitting." + .into(), + ), + kind: SettingKind::I64Slider { + min: 1, + max: 8, + default: i64::try_from(self.settings.stable_frames).unwrap_or(2), + suffix: Some(" frames".into()), + }, + }, + SettingItem { + key: "show_provisional".into(), + label: "Show provisional".into(), + tooltip: Some( + "Show still-growing clusters in the preview overlay and boundary layer." + .into(), + ), + kind: SettingKind::Bool { + default: self.settings.show_provisional, + }, + }, + ], + }, SettingsSection { label: "Refinement".into(), description: Some( - "Frame-based mode and eigenfeature filtering use these thresholds to reject broad or anisotropic clusters." + "Frame-based mode, eigenfeature filtering, and preview overlays use these thresholds and display controls." .into(), ), default_open: false, @@ -427,12 +881,26 @@ impl Plugin for EveSmlmCandidatePlugin { }, SettingItem { key: "show_overlay".into(), - label: "Show overlay".into(), - tooltip: Some("Highlight candidate centroids on the preview.".into()), + label: "Show centroids".into(), + tooltip: Some( + "Draw clickable centroid markers that link into the accepted candidate-events dataset." + .into(), + ), kind: SettingKind::Bool { default: self.settings.show_overlay, }, }, + SettingItem { + key: "show_boundaries".into(), + label: "Show boundaries".into(), + tooltip: Some( + "Draw 2-sigma eigenfeature ellipses or bounding boxes around visible clusters." + .into(), + ), + kind: SettingKind::Bool { + default: self.settings.show_boundaries, + }, + }, ], }, ], @@ -445,71 +913,99 @@ impl Plugin for EveSmlmCandidatePlugin { "polarity" => Some(json!(self.settings.polarity.index())), "epsilon_px" => Some(json!(self.settings.epsilon_px)), "min_events" => Some(json!(self.settings.min_events)), + "lookback_us" => Some(json!(self.settings.lookback_us)), + "stable_frames" => Some(json!(self.settings.stable_frames)), "max_spatial_extent_px" => Some(json!(self.settings.max_spatial_extent_px)), "min_isotropy" => Some(json!(self.settings.min_isotropy)), "threshold_factor" => Some(json!(self.settings.threshold_factor)), "fit_radius_px" => Some(json!(self.settings.fit_radius_px)), "max_candidates" => Some(json!(self.settings.max_candidates)), "show_overlay" => Some(json!(self.settings.show_overlay)), + "show_boundaries" => Some(json!(self.settings.show_boundaries)), + "show_provisional" => Some(json!(self.settings.show_provisional)), _ => None, } } fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + let mut reset_tracking = false; match key { "finding_method" => { let Some(value) = Self::parse_usize(value) else { return Err("finding_method must be an integer".into()); }; self.settings.finding_method = CandidateFindingMethod::from_index(value); + reset_tracking = true; } "polarity" => { let Some(value) = Self::parse_usize(value) else { return Err("polarity must be an integer".into()); }; self.settings.polarity = PolarityMode::from_index(value); + reset_tracking = true; } "epsilon_px" => { let Some(value) = value.as_f64() else { return Err("epsilon_px must be numeric".into()); }; self.settings.epsilon_px = value.clamp(1.0, 10.0); + reset_tracking = true; } "min_events" => { let Some(value) = Self::parse_usize(value) else { return Err("min_events must be an integer".into()); }; self.settings.min_events = value.clamp(1, 64); + reset_tracking = true; + } + "lookback_us" => { + let Some(value) = value.as_u64() else { + return Err("lookback_us must be an integer".into()); + }; + self.settings.lookback_us = value.min(500_000); + reset_tracking = true; + } + "stable_frames" => { + let Some(value) = Self::parse_usize(value) else { + return Err("stable_frames must be an integer".into()); + }; + self.settings.stable_frames = value.clamp(1, 8); + reset_tracking = true; } "max_spatial_extent_px" => { let Some(value) = value.as_f64() else { return Err("max_spatial_extent_px must be numeric".into()); }; self.settings.max_spatial_extent_px = value.clamp(1.0, 20.0); + reset_tracking = true; } "min_isotropy" => { let Some(value) = value.as_f64() else { return Err("min_isotropy must be numeric".into()); }; self.settings.min_isotropy = value.clamp(0.0, 1.0); + reset_tracking = true; } "threshold_factor" => { let Some(value) = value.as_f64() else { return Err("threshold_factor must be numeric".into()); }; self.settings.threshold_factor = value.clamp(0.5, 6.0); + reset_tracking = true; } "fit_radius_px" => { let Some(value) = Self::parse_usize(value) else { return Err("fit_radius_px must be an integer".into()); }; self.settings.fit_radius_px = value.clamp(1, 16); + reset_tracking = true; } "max_candidates" => { let Some(value) = Self::parse_usize(value) else { return Err("max_candidates must be an integer".into()); }; self.settings.max_candidates = value.clamp(1, 2048); + reset_tracking = true; } "show_overlay" => { let Some(value) = value.as_bool() else { @@ -517,14 +1013,30 @@ impl Plugin for EveSmlmCandidatePlugin { }; self.settings.show_overlay = value; } + "show_boundaries" => { + let Some(value) = value.as_bool() else { + return Err("show_boundaries must be a boolean".into()); + }; + self.settings.show_boundaries = value; + } + "show_provisional" => { + let Some(value) = value.as_bool() else { + return Err("show_provisional must be a boolean".into()); + }; + self.settings.show_provisional = value; + } _ => return Err(format!("unknown setting: {key}")), } + if reset_tracking { + self.reset_tracking_state(); + } + Ok(()) } fn status_entries(&self) -> Vec { - vec![ + let mut entries = vec![ StatusEntry::Text(self.last_status.clone()), StatusEntry::LabeledValue { label: "Events".into(), @@ -532,16 +1044,508 @@ impl Plugin for EveSmlmCandidatePlugin { color: None, }, StatusEntry::LabeledValue { - label: "Candidates".into(), + label: "Published".into(), value: self.last_candidate_count.to_string(), color: None, }, + StatusEntry::LabeledValue { + label: "Complete".into(), + value: self.last_complete_visible_count.to_string(), + color: None, + }, + StatusEntry::LabeledValue { + label: "Provisional".into(), + value: self.last_provisional_count.to_string(), + color: None, + }, StatusEntry::LabeledValue { label: "Method".into(), value: self.settings.finding_method.label().into(), color: None, }, - ] + ]; + if self.settings.lookback_us > 0 { + entries.push(StatusEntry::LabeledValue { + label: "Lookback".into(), + value: format!("{} us", self.settings.lookback_us), + color: None, + }); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + candidate_event_registry(&self.current_event_datasets) + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + let dataset = match dataset_id { + ACCEPTED_EVENTS_DATASET_ID => { + candidate_events_dataset(&self.current_event_datasets.accepted) + } + REJECTED_EVENTS_DATASET_ID => { + candidate_events_dataset(&self.current_event_datasets.rejected) + } + _ => return None, + }; + serde_json::to_vec(&dataset).ok() + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + ACCEPTED_EVENTS_DATASET_ID | REJECTED_EVENTS_DATASET_ID => self.dataset_generation, + _ => 0, + } + } +} + +fn boundary_label(method: CandidateFindingMethod) -> &'static str { + match method { + CandidateFindingMethod::FrameBased => "bounding boxes", + CandidateFindingMethod::Dbscan | CandidateFindingMethod::Eigenfeature => { + "2-sigma eigenfeature ellipses" + } + } +} + +fn boundary_pixels( + clusters: &[&EveCluster], + width: u16, + height: u16, +) -> (Vec, Vec) { + let mut complete = HashSet::new(); + let mut provisional = HashSet::new(); + + for cluster in clusters { + let target = if cluster.complete { + &mut complete + } else { + &mut provisional + }; + rasterize_cluster_boundary( + cluster.boundary.as_ref(), + width, + height, + !cluster.complete, + target, + ); + } + + let to_pixels = |points: HashSet<(u16, u16)>| { + let mut pixels: Vec<_> = points.into_iter().map(|(x, y)| FfiPixel { x, y }).collect(); + pixels.sort_by_key(|pixel| (pixel.y, pixel.x)); + pixels + }; + + (to_pixels(complete), to_pixels(provisional)) +} + +fn rasterize_cluster_boundary( + boundary: Option<&ClusterBoundary>, + width: u16, + height: u16, + dashed: bool, + out: &mut HashSet<(u16, u16)>, +) { + let Some(boundary) = boundary else { + return; + }; + + match boundary { + ClusterBoundary::BoundingBox { + x_min, + x_max, + y_min, + y_max, + } => { + for (step, x) in (*x_min..=*x_max).enumerate() { + if !dashed || step % 2 == 0 { + push_boundary_pixel(out, width, height, x as f64, f64::from(*y_min)); + push_boundary_pixel(out, width, height, x as f64, f64::from(*y_max)); + } + } + for (step, y) in (*y_min..=*y_max).enumerate() { + if !dashed || step % 2 == 0 { + push_boundary_pixel(out, width, height, f64::from(*x_min), y as f64); + push_boundary_pixel(out, width, height, f64::from(*x_max), y as f64); + } + } + } + ClusterBoundary::Ellipse { + cx, + cy, + semi_major, + semi_minor, + angle_rad, + } => { + let steps = ((semi_major.max(*semi_minor) * 10.0).ceil() as usize).clamp(24, 240); + let cos_angle = angle_rad.cos(); + let sin_angle = angle_rad.sin(); + for step in 0..=steps { + if dashed && step % 2 == 1 { + continue; + } + let theta = std::f64::consts::TAU * step as f64 / steps as f64; + let ellipse_x = semi_major * theta.cos(); + let ellipse_y = semi_minor * theta.sin(); + let rotated_x = ellipse_x * cos_angle - ellipse_y * sin_angle; + let rotated_y = ellipse_x * sin_angle + ellipse_y * cos_angle; + push_boundary_pixel(out, width, height, cx + rotated_x, cy + rotated_y); + } + } + } +} + +fn push_boundary_pixel(out: &mut HashSet<(u16, u16)>, width: u16, height: u16, x: f64, y: f64) { + let x = x.round(); + let y = y.round(); + if x < 0.0 || y < 0.0 { + return; + } + + let x = x as u16; + let y = y as u16; + if x < width && y < height { + out.insert((x, y)); + } +} + +fn candidate_event_registry(datasets: &CandidateEventDatasets) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: ACCEPTED_EVENTS_DATASET_ID.into(), + title: "Accepted EVE events".into(), + kind: HostDatasetKind::TableV1(candidate_events_schema( + datasets, + ACCEPTED_EVENTS_LAYER_ID, + "accepted candidate events", + "cluster_id", + )), + empty_message: "No accepted candidate events in the current analysis window." + .into(), + display: Some(candidate_event_display_metadata( + "Accepted candidate events", + ACCEPTED_EVENTS_COLOR, + )), + relations: vec![HostDatasetRelation { + target_dataset_id: "augur.evesmlm.current_localizations".into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }, + HostDatasetDescriptor { + id: REJECTED_EVENTS_DATASET_ID.into(), + title: "Rejected EVE events".into(), + kind: HostDatasetKind::TableV1(candidate_events_schema( + datasets, + REJECTED_EVENTS_LAYER_ID, + "rejected candidate events", + "event_id", + )), + empty_message: "No rejected candidate events in the current analysis window." + .into(), + display: Some(candidate_event_display_metadata( + "Rejected candidate events", + REJECTED_EVENTS_COLOR, + )), + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: ACCEPTED_EVENTS_COMPACT_VIEW_ID.into(), + title: "Accepted Events".into(), + dataset_id: ACCEPTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: REJECTED_EVENTS_COMPACT_VIEW_ID.into(), + title: "Rejected Events".into(), + dataset_id: REJECTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: ACCEPTED_EVENTS_TABLE_VIEW_ID.into(), + title: "Accepted Events".into(), + dataset_id: ACCEPTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, + HostViewDescriptor { + id: REJECTED_EVENTS_TABLE_VIEW_ID.into(), + title: "Rejected Events".into(), + dataset_id: REJECTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, + HostViewDescriptor { + id: ACCEPTED_EVENTS_3D_VIEW_ID.into(), + title: "Accepted Events 3D".into(), + dataset_id: ACCEPTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + HostViewDescriptor { + id: REJECTED_EVENTS_3D_VIEW_ID.into(), + title: "Rejected Events 3D".into(), + dataset_id: REJECTED_EVENTS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), + } +} + +fn candidate_event_display_metadata( + layer_title: &str, + color: [u8; 4], +) -> HostDatasetDisplayMetadata { + HostDatasetDisplayMetadata { + layer_title: Some(layer_title.into()), + default_visibility: Some(true), + default_color: Some(color), + default_marker_shape: Some(HostMarkerShape::Point), + default_size: Some(2.5), + } +} + +fn candidate_events_schema( + datasets: &CandidateEventDatasets, + layer_id: &str, + semantic_label: &str, + row_id_column: &str, +) -> TableSchema { + TableSchema { + columns: vec![ + TableColumn { + id: "event_id".into(), + title: "Event ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "x_px".into(), + title: "X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "y_px".into(), + title: "Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "polarity".into(), + title: "Polarity".into(), + value_type: TableValueType::Bool, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: datasets + .sensor_dims + .map(|(width, height)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min: 0.0, + x_max: f64::from(width), + y_min: 0.0, + y_max: f64::from(height), + }), + coordinate_space_3d: datasets + .sensor_dims + .map(|(width, height)| TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min: 0.0, + x_max: f64::from(width), + y_min: 0.0, + y_max: f64::from(height), + z_min: datasets.frame_window_start_us as f64, + z_max: datasets + .frame_window_end_us + .max(datasets.frame_window_start_us) as f64, + }), + row_id_column: Some(row_id_column.into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(layer_id.into()), + semantic_label: Some(semantic_label.into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("timestamp_us".into()), + span_end_column: Some("timestamp_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "event_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: true, + label: None, + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + width_priority: Some(TableColumnWidthPriority::Medium), + hide_in_compact: false, + label: Some("Time".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: false, + label: Some("X".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: false, + label: Some("Y".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "polarity".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + width_priority: Some(TableColumnWidthPriority::Low), + hide_in_compact: false, + label: Some("Polarity".into()), + headline: false, + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + width_priority: Some(TableColumnWidthPriority::Medium), + hide_in_compact: false, + label: Some("Cluster".into()), + headline: row_id_column == "cluster_id", + }, + }, + ], + } +} + +fn candidate_events_dataset(rows: &[CandidateEventRow]) -> TableDatasetV1 { + TableDatasetV1::new(vec![ + TableColumnData { + column_id: "event_id".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.event_id).collect()), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.timestamp_us).collect()), + }, + TableColumnData { + column_id: "x_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.x_px).collect()), + }, + TableColumnData { + column_id: "y_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.y_px).collect()), + }, + TableColumnData { + column_id: "polarity".into(), + values: TableColumnValues::Bool(rows.iter().map(|row| row.polarity).collect()), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::String( + rows.iter().map(|row| row.cluster_id.clone()).collect(), + ), + }, + ]) + .expect("candidate event columns must stay aligned") +} + +fn candidate_event_row_id(event: &EveEvent, occurrence: u32) -> u64 { + event.timestamp + ^ u64::from(event.x).rotate_left(11) + ^ u64::from(event.y).rotate_left(23) + ^ u64::from(event.polarity as u8).rotate_left(37) + ^ u64::from(occurrence).rotate_left(47) +} + +fn build_candidate_event_datasets( + sensor_dims: (u16, u16), + frame_window_start_us: u64, + frame_window_end_us: u64, + events: &[EveEvent], + cluster_indices: &[Vec], + visible_clusters: &[EveCluster], +) -> CandidateEventDatasets { + let mut cluster_by_event = vec![None; events.len()]; + for (cluster, indices) in visible_clusters.iter().zip(cluster_indices.iter()) { + for &event_index in indices { + if let Some(slot) = cluster_by_event.get_mut(event_index) { + *slot = Some(cluster.cluster_id.to_string()); + } + } + } + + let mut accepted = Vec::new(); + let mut rejected = Vec::new(); + let mut seen_occurrences = HashMap::new(); + for (event_index, event) in events.iter().enumerate() { + let occurrence = seen_occurrences + .entry((event.timestamp, event.x, event.y, event.polarity)) + .or_insert(0u32); + let row = CandidateEventRow { + event_id: candidate_event_row_id(event, *occurrence), + x_px: f64::from(event.x), + y_px: f64::from(event.y), + timestamp_us: event.timestamp, + polarity: event.polarity, + cluster_id: cluster_by_event[event_index].clone().unwrap_or_default(), + }; + *occurrence = occurrence.saturating_add(1); + if cluster_by_event[event_index].is_some() { + accepted.push(row); + } else { + rejected.push(row); + } + } + + CandidateEventDatasets { + accepted, + rejected, + sensor_dims: Some(sensor_dims), + frame_window_start_us, + frame_window_end_us, } } @@ -555,7 +1559,46 @@ fn empty_candidates(frame: &PluginFrame<'_>, method: CandidateFindingMethod) -> } } -fn clusters_from_indices(events: &[EveEvent], cluster_indices: Vec>) -> Vec { +#[allow(clippy::too_many_arguments)] +fn cluster_boundary_for_indices( + events: &[EveEvent], + indices: &[usize], + centroid_x: f64, + centroid_y: f64, + x_min: u16, + x_max: u16, + y_min: u16, + y_max: u16, + method: CandidateFindingMethod, +) -> ClusterBoundary { + if matches!( + method, + CandidateFindingMethod::Dbscan | CandidateFindingMethod::Eigenfeature + ) { + if let Some(info) = eigenfeature::cluster_eigen_info(events, indices) { + return ClusterBoundary::Ellipse { + cx: centroid_x, + cy: centroid_y, + semi_major: (2.0 * info.lambda_1.max(0.0).sqrt()).max(1.0), + semi_minor: (2.0 * info.lambda_2.max(0.0).sqrt()).max(1.0), + angle_rad: info.angle_rad, + }; + } + } + + ClusterBoundary::BoundingBox { + x_min, + x_max, + y_min, + y_max, + } +} + +fn clusters_from_indices( + events: &[EveEvent], + cluster_indices: Vec>, + method: CandidateFindingMethod, +) -> Vec { cluster_indices .into_iter() .filter_map(|indices| { @@ -572,7 +1615,7 @@ fn clusters_from_indices(events: &[EveEvent], cluster_indices: Vec>) let mut y_min = u16::MAX; let mut y_max = 0; - for index in indices { + for &index in &indices { let event = events[index]; cluster_events.push(event); sum_x += f64::from(event.x); @@ -597,15 +1640,24 @@ fn clusters_from_indices(events: &[EveEvent], cluster_indices: Vec>) .collect(); histogram_entries.sort_by_key(|entry| (entry.1, entry.0)); + let centroid_x = sum_x / count; + let centroid_y = sum_y / count; + let boundary = cluster_boundary_for_indices( + events, &indices, centroid_x, centroid_y, x_min, x_max, y_min, y_max, method, + ); + Some(EveCluster { + cluster_id: 0, pixel_histogram: histogram_entries, events: cluster_events, - centroid_x: sum_x / count, - centroid_y: sum_y / count, + centroid_x, + centroid_y, x_min, x_max, y_min, y_max, + complete: false, + boundary: Some(boundary), }) }) .collect() @@ -871,7 +1923,11 @@ mod tests { event(10, 11, true, 4), ]; - let clusters = clusters_from_indices(&events, vec![vec![0, 1, 2, 3]]); + let clusters = clusters_from_indices( + &events, + vec![vec![0, 1, 2, 3]], + CandidateFindingMethod::Dbscan, + ); assert_eq!(clusters.len(), 1); let cluster = &clusters[0]; assert_eq!(cluster.event_count(), 4); @@ -880,6 +1936,7 @@ mod tests { assert_eq!(cluster.pixel_histogram.len(), 3); assert!((cluster.centroid_x - 10.25).abs() < 1e-6); assert!((cluster.centroid_y - 10.25).abs() < 1e-6); + assert!(cluster.boundary.is_some()); } #[test] @@ -892,7 +1949,7 @@ mod tests { }; let image = build_analysis_image(&frame.into_plugin_frame(), &events); - let index = 1usize * 6 + 2usize; + let index = 6usize + 2usize; assert_eq!(image[index], 20.0); } @@ -909,6 +1966,139 @@ mod tests { assert!(maxima.iter().any(|(x, y, _)| (*x, *y) == (3, 3))); } + #[test] + fn candidate_event_datasets_split_accepted_and_rejected_events() { + let events = vec![ + event(10, 10, true, 101), + event(11, 10, true, 102), + event(12, 10, false, 103), + event(30, 20, true, 104), + ]; + let visible_clusters = vec![EveCluster { + cluster_id: 42, + pixel_histogram: vec![(10, 10, 1, 0), (12, 10, 0, 1)], + events: vec![events[0], events[2]], + centroid_x: 11.0, + centroid_y: 10.0, + x_min: 10, + x_max: 12, + y_min: 10, + y_max: 10, + complete: false, + boundary: Some(ClusterBoundary::BoundingBox { + x_min: 10, + x_max: 12, + y_min: 10, + y_max: 10, + }), + }]; + + let datasets = build_candidate_event_datasets( + (32, 24), + 100, + 101, + &events, + &[vec![0, 2]], + &visible_clusters, + ); + assert_eq!(datasets.accepted.len(), 2); + assert_eq!(datasets.rejected.len(), 2); + assert_eq!(datasets.accepted[0].cluster_id, "42"); + assert_eq!(datasets.rejected[0].cluster_id, ""); + } + + #[test] + fn candidate_event_registry_exposes_table_and_3d_views() { + let registry = candidate_event_registry(&CandidateEventDatasets { + accepted: Vec::new(), + rejected: Vec::new(), + sensor_dims: Some((128, 64)), + frame_window_start_us: 10, + frame_window_end_us: 20, + }); + assert_eq!(registry.datasets.len(), 2); + assert_eq!(registry.views.len(), 6); + assert_eq!(registry.views[0].id, ACCEPTED_EVENTS_COMPACT_VIEW_ID); + assert_eq!(registry.views[0].title, "Accepted Events"); + assert!(matches!(registry.views[0].kind, HostViewKind::CompactTable)); + assert_eq!(registry.views[2].id, ACCEPTED_EVENTS_TABLE_VIEW_ID); + assert_eq!(registry.views[2].title, "Accepted Events"); + assert!(matches!(registry.views[2].kind, HostViewKind::TableWindow)); + assert_eq!(registry.views[4].id, ACCEPTED_EVENTS_3D_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.row_id_column.as_deref(), Some("cluster_id")); + let cluster_column = schema.column("cluster_id").expect("cluster id column"); + assert_eq!(cluster_column.value_type, TableValueType::String); + assert_eq!( + schema + .column_display("cluster_id") + .map(|display| display.headline), + Some(true) + ); + assert_eq!( + schema + .coordinate_space_3d + .as_ref() + .map(|space| space.z_column.as_str()), + Some("timestamp_us") + ); + let rejected_schema = match ®istry.datasets[1].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(rejected_schema.row_id_column.as_deref(), Some("event_id")); + } + + #[test] + fn temporal_tracking_waits_for_stable_frames_before_publishing() { + let mut plugin = EveSmlmCandidatePlugin::default(); + plugin.settings.stable_frames = 2; + + let make_cluster = || EveCluster { + cluster_id: 0, + pixel_histogram: vec![(10, 10, 3, 0), (11, 10, 2, 0)], + events: vec![ + event(10, 10, true, 1), + event(10, 10, true, 2), + event(10, 10, true, 3), + event(11, 10, true, 4), + event(11, 10, true, 5), + ], + centroid_x: 10.4, + centroid_y: 10.0, + x_min: 10, + x_max: 11, + y_min: 10, + y_max: 10, + complete: false, + boundary: Some(ClusterBoundary::BoundingBox { + x_min: 10, + x_max: 11, + y_min: 10, + y_max: 10, + }), + }; + + let (visible, published) = plugin.update_tracked_clusters(vec![make_cluster()], true); + assert_eq!(visible.len(), 1); + assert!(!visible[0].complete); + assert!(published.is_empty()); + + let (visible, published) = plugin.update_tracked_clusters(vec![make_cluster()], true); + assert_eq!(visible.len(), 1); + assert!(!visible[0].complete); + assert!(published.is_empty()); + + let (visible, published) = plugin.update_tracked_clusters(vec![make_cluster()], true); + assert_eq!(visible.len(), 1); + assert!(visible[0].complete); + assert_eq!(published.len(), 1); + assert_eq!(published[0].cluster_id, visible[0].cluster_id); + } + struct TestFrame { width: u16, height: u16, diff --git a/plugins/evesmlm-candidates/src/types.rs b/plugins/evesmlm-candidates/src/types.rs index 1f1fbd1..6bf8b1b 100644 --- a/plugins/evesmlm-candidates/src/types.rs +++ b/plugins/evesmlm-candidates/src/types.rs @@ -3,6 +3,10 @@ use serde::{Deserialize, Serialize}; pub const CTX_EVE_CANDIDATES: &str = "augur.evesmlm.candidates"; +fn default_cluster_complete() -> bool { + true +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum CandidateFindingMethod { @@ -33,7 +37,7 @@ pub struct EveEvent { impl From for EveEvent { fn from(value: FfiCdEvent) -> Self { Self { - timestamp: value.timestamp, + timestamp: value.timestamp_us(), x: value.x, y: value.y, polarity: value.polarity != 0, @@ -41,8 +45,28 @@ impl From for EveEvent { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ClusterBoundary { + BoundingBox { + x_min: u16, + x_max: u16, + y_min: u16, + y_max: u16, + }, + Ellipse { + cx: f64, + cy: f64, + semi_major: f64, + semi_minor: f64, + angle_rad: f64, + }, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EveCluster { + #[serde(default)] + pub cluster_id: u64, /// Per-pixel event histogram: (x, y, n_positive, n_negative) pub pixel_histogram: Vec<(u16, u16, u32, u32)>, /// All raw events assigned to this cluster. @@ -55,6 +79,10 @@ pub struct EveCluster { pub x_max: u16, pub y_min: u16, pub y_max: u16, + #[serde(default = "default_cluster_complete")] + pub complete: bool, + #[serde(default)] + pub boundary: Option, } impl EveCluster { @@ -96,3 +124,17 @@ pub struct EveCandidates { pub n_events_processed: usize, pub finding_method: CandidateFindingMethod, } + +#[derive(Debug, Clone)] +pub(crate) struct TrackedCluster { + pub id: u64, + pub centroid_x: f64, + pub centroid_y: f64, + pub event_count: usize, + pub last_seen_frame: u64, + pub last_grown_frame: u64, + pub frames_since_growth: usize, + pub complete: bool, + pub emitted: bool, + pub cluster: EveCluster, +} diff --git a/plugins/evesmlm-fitting/README.md b/plugins/evesmlm-fitting/README.md index 7fdf312..ba5be95 100644 --- a/plugins/evesmlm-fitting/README.md +++ b/plugins/evesmlm-fitting/README.md @@ -23,6 +23,7 @@ Sub-pixel localization for eveSMLM candidate clusters. The plugin consumes `EveC | Sigma max | `200.0` nm | Upper accepted sigma bound for sigma-producing methods | | Max fit residual | `0.5` | Reject fits above this residual | | Show overlay | `true` | Highlight accepted localization positions | +| Show rejected | `false` | Draw rejected fits as linked diamond markers | AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `GlobalSettings`. This plugin uses the host `nm_per_pixel` value automatically for sigma filtering when it is available, while retaining a hidden fallback for older hosts. @@ -34,11 +35,24 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global - `EveLocalizationResults` on `augur.evesmlm.localization_results` - `LocalizationResults` on `augur.localization.results` for compatibility with plugins such as Focus Metrics -- the compact host-view dataset `augur.evesmlm.current_localizations` +- the shared host-view dataset `augur.evesmlm.current_localizations` +- the rejected-fit investigation dataset `augur.evesmlm.rejected_fits` ## Host View -The plugin declares the compact analysis-panel view `augur.evesmlm.current_localizations.compact`. If `EVE Post-Processing` is also enabled, the host resolves that same view id to the later post-processing stage instead. +The plugin declares the shared current-localizations dataset plus: + +- the compact analysis-panel view `augur.evesmlm.current_localizations.compact` +- a linked 3D scatter view over the same dataset +- compact, windowed, and 3D views for rejected fits + +That dataset now carries stable row ids, timestamps, 2D/3D coordinate metadata, and layer/display metadata so the host can keep selection stable across tables, overlays, and 3D inspection. + +Rejected fits are exposed as structured rows with `cluster_id`, timestamps, fit metrics, and a categorical rejection reason so fit failures and threshold rejections can be inspected directly instead of inferred from a counter alone. + +Rejected-fit selection is currently local to the rejected-fits dataset. Matching `cluster_id` values do not create cross-dataset linking back to candidate-event rows because AugurRS stable row keys are scoped by dataset id. + +If `EVE Post-Processing` is also enabled, the host resolves those same dataset/view ids to the later post-processing stage instead. ## Dependencies diff --git a/plugins/evesmlm-fitting/src/lib.rs b/plugins/evesmlm-fitting/src/lib.rs index bd7a370..31b83b3 100644 --- a/plugins/evesmlm-fitting/src/lib.rs +++ b/plugins/evesmlm-fitting/src/lib.rs @@ -3,6 +3,8 @@ //! Consumes `EveCandidates` and localizes each raw-event cluster to //! sub-pixel precision with a configurable fitting backend. +use std::collections::HashMap; + pub mod gaussian; pub mod log_gaussian; pub mod mean_xy; @@ -11,48 +13,140 @@ pub mod radial_symmetry; pub mod types; use augur_plugin_api::{ - export_plugin, AnalysisSeverity, EventStoreHandle, FfiSubpixelMarker, GlobalSettings, + export_plugin, AnalysisSeverity, EventStoreHandle, FfiColorRgba, FfiMarkerOverlayItem, + FfiMarkerShape, GlobalSettings, HostActionDescriptor, HostActionRequestQueue, HostActionScope, HostContext, HostOutput, Plugin, PluginFrame, PluginInput, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, CTX_GLOBAL_SETTINGS, + CTX_INVESTIGATION_ACTION_REQUESTS, HOST_ACTION_CLUSTER_ROWS_PARAM, }; use augur_plugin_api::{ - HostDatasetDescriptor, HostDatasetKind, HostViewDescriptor, HostViewKind, HostViewPlacement, - HostViewRegistry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, + HostDatasetDescriptor, HostDatasetDisplayMetadata, HostDatasetKind, HostDatasetRelation, + HostMarkerShape, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, + TableColumn, TableColumnData, TableColumnDisplayEntry, TableColumnDisplayFormat, + TableColumnDisplayMetadata, TableColumnValues, TableCoordinateSpace2d, TableCoordinateSpace3d, + TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, }; pub use augur_plugin_evesmlm_candidates::{ - CandidateFindingMethod, EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, + EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, }; use augur_plugin_types::{Localization, LocalizationResults, CTX_LOCALIZATION_RESULTS}; use serde_json::{json, Value}; -pub use types::{EveLocalization, EveLocalizationResults, FitMethod, CTX_EVE_LOCALIZATION_RESULTS}; +pub use types::{ + EveLocalization, EveLocalizationResults, FitMethod, RejectedFitRow, RejectionReason, + CTX_EVE_LOCALIZATION_RESULTS, +}; const OVERLAY_COLOR: [u8; 4] = [60, 220, 140, 220]; const CANDIDATE_DEPENDENCY: [&str; 1] = ["EVE Candidate Finding"]; pub const CURRENT_LOCALIZATIONS_DATASET_ID: &str = "augur.evesmlm.current_localizations"; +pub const CURRENT_LOCALIZATIONS_LAYER_ID: &str = "augur.layer.evesmlm.current_localizations"; pub const CURRENT_LOCALIZATIONS_VIEW_ID: &str = "augur.evesmlm.current_localizations.compact"; +pub const CURRENT_LOCALIZATIONS_3D_VIEW_ID: &str = "augur.evesmlm.current_localizations.scatter3d"; +pub const REJECTED_FITS_DATASET_ID: &str = "augur.evesmlm.rejected_fits"; +pub const REJECTED_FITS_LAYER_ID: &str = "augur.layer.evesmlm.rejected_fits"; +pub const REJECTED_FITS_COMPACT_VIEW_ID: &str = "augur.evesmlm.rejected_fits.compact"; +pub const REJECTED_FITS_TABLE_VIEW_ID: &str = "augur.evesmlm.rejected_fits.table"; +pub const REJECTED_FITS_3D_VIEW_ID: &str = "augur.evesmlm.rejected_fits.scatter3d"; + +pub const REFIT_PREVIEW_DATASET_ID: &str = "augur.evesmlm.refit_preview"; +pub const REFIT_PREVIEW_LAYER_ID: &str = "augur.layer.evesmlm.refit_preview"; +pub const REFIT_PREVIEW_VIEW_ID: &str = "augur.evesmlm.refit_preview.compact"; + +pub const ACCEPTED_CANDIDATE_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; + +pub const ACTION_REFIT_CLUSTER: &str = "augur.evesmlm.refit_cluster"; +pub const ACTION_COMMIT_REFIT: &str = "augur.evesmlm.commit_refit"; +pub const ACTION_DISCARD_REFIT: &str = "augur.evesmlm.discard_refit"; pub fn current_localizations_registry() -> HostViewRegistry { + current_localizations_registry_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_registry_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> HostViewRegistry { HostViewRegistry { datasets: vec![HostDatasetDescriptor { id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), title: "Current EVE localizations".into(), - kind: HostDatasetKind::TableV1(current_localizations_schema()), + kind: HostDatasetKind::TableV1(current_localizations_schema_for_results( + results, + sensor_dims, + )), empty_message: "No EVE localizations in the current frame.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Current EVE localizations".into()), + default_visibility: Some(true), + default_color: Some([90, 170, 255, 255]), + default_marker_shape: Some(HostMarkerShape::Cross), + default_size: Some(6.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], }], - views: vec![HostViewDescriptor { - id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), - title: "Current Localizations".into(), - dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }], + views: vec![ + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), + title: "Current Localizations".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_3D_VIEW_ID.into(), + title: "Current Localizations 3D".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), } } pub fn current_localizations_schema() -> TableSchema { + current_localizations_schema_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_schema_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> TableSchema { TableSchema { columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_start_us".into(), + title: "Span Start (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_end_us".into(), + title: "Span End (us)".into(), + value_type: TableValueType::U64, + }, TableColumn { id: "x_px".into(), title: "X (px)".into(), @@ -78,13 +172,172 @@ pub fn current_localizations_schema() -> TableSchema { title: "Events".into(), value_type: TableValueType::U64, }, + TableColumn { + id: "polarity_balance".into(), + title: "Polarity balance".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_residual".into(), + title: "Fit residual".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_method".into(), + title: "Fit method".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: current_localizations_2d_space(results, sensor_dims), + coordinate_space_3d: current_localizations_3d_space(results, sensor_dims), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(CURRENT_LOCALIZATIONS_LAYER_ID.into()), + semantic_label: Some("localizations".into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("span_start_us".into()), + span_end_column: Some("span_end_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "row_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_start_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span start".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_end_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span end".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_residual".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_method".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + ..Default::default() + }, + }, ], - coordinate_space_2d: None, } } pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { TableDatasetV1::new(vec![ + TableColumnData { + column_id: "row_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(localization_row_id) + .collect(), + ), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.cluster_id) + .collect(), + ), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.timestamp_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_start_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_start_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_end_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_end_us) + .collect(), + ), + }, TableColumnData { column_id: "x_px".into(), values: TableColumnValues::F64( @@ -127,187 +380,1543 @@ pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableD .collect(), ), }, + TableColumnData { + column_id: "polarity_balance".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.polarity_balance) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_residual".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.fit_residual) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_method".into(), + values: TableColumnValues::String( + results + .localizations + .iter() + .map(|value| value.fit_method.label().to_owned()) + .collect(), + ), + }, ]) .expect("current localization columns should stay aligned") } -#[derive(Debug, Clone, Copy)] -pub(crate) struct FitEstimate { - pub x: f64, - pub y: f64, - pub sigma_x: f64, - pub sigma_y: f64, - pub residual: f64, +fn current_localizations_2d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results)) + .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min, + x_max, + y_min, + y_max, + }) } -#[derive(Debug, Clone)] -pub struct FittingSettings { - pub fit_method: FitMethod, - pub nm_per_pixel: f64, - pub sigma_min_nm: f64, - pub sigma_max_nm: f64, - pub max_fit_residual: f64, - pub show_overlay: bool, +fn current_localizations_3d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + let (x_min, x_max, y_min, y_max) = sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results))?; + let (z_min, z_max) = localization_time_bounds(results)?; + Some(TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + }) } -impl Default for FittingSettings { - fn default() -> Self { - Self { - fit_method: FitMethod::LogGaussian, - nm_per_pixel: 65.0, - sigma_min_nm: 80.0, - sigma_max_nm: 200.0, - max_fit_residual: 0.5, - show_overlay: true, - } +pub fn refit_preview_registry_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: REFIT_PREVIEW_DATASET_ID.into(), + title: "Refit preview".into(), + kind: HostDatasetKind::TableV1(refit_preview_schema(results, sensor_dims)), + empty_message: "No pending re-fit preview.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Refit preview".into()), + default_visibility: Some(true), + default_color: Some([255, 210, 90, 240]), + default_marker_shape: Some(HostMarkerShape::Circle), + default_size: Some(8.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }], + views: vec![HostViewDescriptor { + id: REFIT_PREVIEW_VIEW_ID.into(), + title: "Refit Preview".into(), + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }], + actions: Vec::new(), } } -pub struct EveSmlmFittingPlugin { - enabled: bool, - settings: FittingSettings, - current_results: EveLocalizationResults, - last_localization_count: usize, - last_rejection_count: usize, - last_status: String, - dataset_generation: u64, +pub fn refit_preview_schema( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> TableSchema { + let mut schema = current_localizations_schema_for_results(results, sensor_dims); + schema.layer_id = Some(REFIT_PREVIEW_LAYER_ID.into()); + schema.semantic_label = Some("refit preview".into()); + schema } -impl Default for EveSmlmFittingPlugin { - fn default() -> Self { - Self { - enabled: false, - settings: FittingSettings::default(), - current_results: EveLocalizationResults::default(), - last_localization_count: 0, - last_rejection_count: 0, - last_status: - "Enable the plugin to fit EVE candidate clusters to sub-pixel localizations.".into(), - dataset_generation: 0, +pub fn refit_preview_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { + current_localizations_dataset(results) +} + +fn refit_action_param_schema() -> SettingsSchema { + SettingsSchema { + sections: vec![SettingsSection { + label: "Refit parameters".into(), + description: Some( + "Re-run the chosen cluster's fit with these parameters and preview the result before committing." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "fit_method".into(), + label: "Method".into(), + tooltip: Some("Fitting backend to use for this cluster.".into()), + kind: SettingKind::Enum { + variants: vec![ + FitMethod::LogGaussian.label().into(), + FitMethod::Gaussian.label().into(), + FitMethod::RadialSymmetry.label().into(), + FitMethod::Phasor.label().into(), + FitMethod::MeanXY.label().into(), + ], + default: FitMethod::LogGaussian.index(), + }, + }, + SettingItem { + key: "sigma_min_nm".into(), + label: "Sigma min".into(), + tooltip: Some("Reject fits with sigma below this bound.".into()), + kind: SettingKind::F64Slider { + min: 10.0, + max: 500.0, + default: FittingSettings::default().sigma_min_nm, + suffix: Some(" nm".into()), + }, + }, + SettingItem { + key: "sigma_max_nm".into(), + label: "Sigma max".into(), + tooltip: Some("Reject fits with sigma above this bound.".into()), + kind: SettingKind::F64Slider { + min: 10.0, + max: 500.0, + default: FittingSettings::default().sigma_max_nm, + suffix: Some(" nm".into()), + }, + }, + SettingItem { + key: "max_fit_residual".into(), + label: "Max residual".into(), + tooltip: Some("Reject fits whose residual exceeds this threshold.".into()), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: FittingSettings::default().max_fit_residual, + }, + }, + ], + }], + } +} + +fn localization_xy_bounds(results: &EveLocalizationResults) -> Option<(f64, f64, f64, f64)> { + let mut localizations = results.localizations.iter(); + let first = localizations.next()?; + let mut x_min = first.x; + let mut x_max = first.x; + let mut y_min = first.y; + let mut y_max = first.y; + for localization in localizations { + x_min = x_min.min(localization.x); + x_max = x_max.max(localization.x); + y_min = y_min.min(localization.y); + y_max = y_max.max(localization.y); + } + Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) +} + +fn localization_time_bounds(results: &EveLocalizationResults) -> Option<(f64, f64)> { + if let Some(first) = results.localizations.first() { + let mut min_time = first.timestamp_us; + let mut max_time = first.timestamp_us; + for localization in &results.localizations { + min_time = min_time.min(localization.timestamp_us); + max_time = max_time.max(localization.timestamp_us); } + return Some((min_time as f64, max_time.max(min_time) as f64)); + } + + if results.frame_window_end_us >= results.frame_window_start_us { + return Some(( + results.frame_window_start_us as f64, + results.frame_window_end_us as f64, + )); } + + None } -impl EveSmlmFittingPlugin { - fn nm_per_pixel(&self, context: &HostContext<'_>) -> f64 { - context - .get::(CTX_GLOBAL_SETTINGS) - .ok() - .flatten() - .map(|settings| settings.nm_per_pixel) - .unwrap_or(self.settings.nm_per_pixel) +pub fn localization_row_id(localization: &EveLocalization) -> u64 { + localization.cluster_id.rotate_left(3) + ^ localization.timestamp_us + ^ localization.x.to_bits().rotate_left(7) + ^ localization.y.to_bits().rotate_left(19) + ^ localization.sigma_x.to_bits().rotate_left(31) + ^ localization.sigma_y.to_bits().rotate_left(43) + ^ localization.fit_residual.to_bits().rotate_left(53) + ^ (localization.n_events as u64).rotate_left(11) + ^ (localization.fit_method.index() as u64).rotate_left(59) + ^ localization.span_start_us.rotate_left(17) + ^ localization.span_end_us.rotate_left(29) +} + +pub fn rejected_fit_row_id(row: &RejectedFitRow) -> u64 { + row.timestamp_us + ^ row.cluster_id.rotate_left(7) + ^ row.x.to_bits().rotate_left(19) + ^ row.y.to_bits().rotate_left(31) + ^ row.fit_residual.to_bits().rotate_left(43) + ^ (row.rejection_reason as u64).rotate_left(53) + ^ row.span_start_us.rotate_left(17) + ^ row.span_end_us.rotate_left(29) +} + +fn rejected_fits_registry( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: REJECTED_FITS_DATASET_ID.into(), + title: "Rejected EVE fits".into(), + kind: HostDatasetKind::TableV1(rejected_fits_schema( + rows, + sensor_dims, + frame_window_start_us, + frame_window_end_us, + )), + empty_message: "No rejected EVE fits in the current analysis window.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Rejected EVE fits".into()), + default_visibility: Some(false), + default_color: Some([255, 90, 90, 200]), + default_marker_shape: Some(HostMarkerShape::Diamond), + default_size: Some(5.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }], + views: vec![ + HostViewDescriptor { + id: REJECTED_FITS_COMPACT_VIEW_ID.into(), + title: "Rejected Fits".into(), + dataset_id: REJECTED_FITS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: REJECTED_FITS_TABLE_VIEW_ID.into(), + title: "Rejected Fits Table".into(), + dataset_id: REJECTED_FITS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, + HostViewDescriptor { + id: REJECTED_FITS_3D_VIEW_ID.into(), + title: "Rejected Fits 3D".into(), + dataset_id: REJECTED_FITS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), } +} + +fn rejected_fits_schema( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +) -> TableSchema { + TableSchema { + columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_start_us".into(), + title: "Span Start (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_end_us".into(), + title: "Span End (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "x_px".into(), + title: "X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "y_px".into(), + title: "Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_x_px".into(), + title: "Sigma X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_y_px".into(), + title: "Sigma Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_residual".into(), + title: "Fit residual".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "n_events".into(), + title: "Events".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "polarity_balance".into(), + title: "Polarity balance".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "rejection_reason".into(), + title: "Rejection reason".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: rejected_fits_2d_space(rows, sensor_dims), + coordinate_space_3d: rejected_fits_3d_space( + rows, + sensor_dims, + frame_window_start_us, + frame_window_end_us, + ), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(REJECTED_FITS_LAYER_ID.into()), + semantic_label: Some("rejected fits".into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("span_start_us".into()), + span_end_column: Some("span_end_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "row_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_start_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span start".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_end_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span end".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_residual".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "rejection_reason".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + headline: true, + ..Default::default() + }, + }, + ], + } +} + +fn rejected_fits_dataset(rows: &[RejectedFitRow]) -> TableDatasetV1 { + TableDatasetV1::new(vec![ + TableColumnData { + column_id: "row_id".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.row_id).collect()), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.cluster_id).collect()), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.timestamp_us).collect()), + }, + TableColumnData { + column_id: "span_start_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.span_start_us).collect()), + }, + TableColumnData { + column_id: "span_end_us".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.span_end_us).collect()), + }, + TableColumnData { + column_id: "x_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.x).collect()), + }, + TableColumnData { + column_id: "y_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.y).collect()), + }, + TableColumnData { + column_id: "sigma_x_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.sigma_x).collect()), + }, + TableColumnData { + column_id: "sigma_y_px".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.sigma_y).collect()), + }, + TableColumnData { + column_id: "fit_residual".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.fit_residual).collect()), + }, + TableColumnData { + column_id: "n_events".into(), + values: TableColumnValues::U64(rows.iter().map(|row| row.n_events).collect()), + }, + TableColumnData { + column_id: "polarity_balance".into(), + values: TableColumnValues::F64(rows.iter().map(|row| row.polarity_balance).collect()), + }, + TableColumnData { + column_id: "rejection_reason".into(), + values: TableColumnValues::String( + rows.iter() + .map(|row| row.rejection_reason.as_str().to_owned()) + .collect(), + ), + }, + ]) + .expect("rejected-fit columns should stay aligned") +} + +fn rejected_fits_2d_space( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, +) -> Option { + sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| rejected_fit_xy_bounds(rows)) + .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min, + x_max, + y_min, + y_max, + }) +} + +fn rejected_fits_3d_space( + rows: &[RejectedFitRow], + sensor_dims: Option<(u16, u16)>, + frame_window_start_us: u64, + frame_window_end_us: u64, +) -> Option { + let (x_min, x_max, y_min, y_max) = sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| rejected_fit_xy_bounds(rows))?; + let (z_min, z_max) = rejected_fit_time_bounds(rows) + .unwrap_or((frame_window_start_us as f64, frame_window_end_us as f64)); + Some(TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + }) +} + +fn rejected_fit_xy_bounds(rows: &[RejectedFitRow]) -> Option<(f64, f64, f64, f64)> { + let mut rows = rows.iter(); + let first = rows.next()?; + let mut x_min = first.x; + let mut x_max = first.x; + let mut y_min = first.y; + let mut y_max = first.y; + for row in rows { + x_min = x_min.min(row.x); + x_max = x_max.max(row.x); + y_min = y_min.min(row.y); + y_max = y_max.max(row.y); + } + Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) +} + +fn rejected_fit_time_bounds(rows: &[RejectedFitRow]) -> Option<(f64, f64)> { + let mut rows = rows.iter(); + let first = rows.next()?; + let mut min_time = first.timestamp_us; + let mut max_time = first.timestamp_us; + for row in rows { + min_time = min_time.min(row.timestamp_us); + max_time = max_time.max(row.timestamp_us); + } + Some((min_time as f64, max_time.max(min_time) as f64)) +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct FitEstimate { + pub x: f64, + pub y: f64, + pub sigma_x: f64, + pub sigma_y: f64, + pub residual: f64, +} + +#[derive(Debug, Clone)] +pub struct FittingSettings { + pub fit_method: FitMethod, + pub nm_per_pixel: f64, + pub sigma_min_nm: f64, + pub sigma_max_nm: f64, + pub max_fit_residual: f64, + pub show_overlay: bool, + pub show_rejected_overlay: bool, +} + +impl Default for FittingSettings { + fn default() -> Self { + Self { + fit_method: FitMethod::LogGaussian, + nm_per_pixel: 65.0, + sigma_min_nm: 80.0, + sigma_max_nm: 200.0, + max_fit_residual: 0.5, + show_overlay: true, + show_rejected_overlay: false, + } + } +} + +pub struct EveSmlmFittingPlugin { + enabled: bool, + settings: FittingSettings, + current_results: EveLocalizationResults, + current_rejected_fits: Vec, + host_results: EveLocalizationResults, + host_rejected_fits: Vec, + sensor_dims: Option<(u16, u16)>, + last_localization_count: usize, + last_rejection_count: usize, + last_fit_failure_count: usize, + last_sigma_rejection_count: usize, + last_residual_rejection_count: usize, + last_status: String, + dataset_generation: u64, + refit_preview_results: EveLocalizationResults, + /// Parallel to `refit_preview_results.localizations`: for each preview + /// row, the `row_id` of the current localization it should replace on + /// commit (or `None` if commit should append). + refit_preview_replaces: Vec>, + last_consumed_action_request_id: u64, + last_action_notice: Option, +} + +impl Default for EveSmlmFittingPlugin { + fn default() -> Self { + Self { + enabled: false, + settings: FittingSettings::default(), + current_results: EveLocalizationResults::default(), + current_rejected_fits: Vec::new(), + host_results: EveLocalizationResults::default(), + host_rejected_fits: Vec::new(), + sensor_dims: None, + last_localization_count: 0, + last_rejection_count: 0, + last_fit_failure_count: 0, + last_sigma_rejection_count: 0, + last_residual_rejection_count: 0, + last_status: + "Enable the plugin to fit EVE candidate clusters to sub-pixel localizations.".into(), + dataset_generation: 0, + refit_preview_results: EveLocalizationResults::default(), + refit_preview_replaces: Vec::new(), + last_consumed_action_request_id: 0, + last_action_notice: None, + } + } +} + +impl EveSmlmFittingPlugin { + fn nm_per_pixel(&self, context: &HostContext<'_>) -> f64 { + context + .get::(CTX_GLOBAL_SETTINGS) + .ok() + .flatten() + .map(|settings| settings.nm_per_pixel) + .unwrap_or(self.settings.nm_per_pixel) + } + + fn sync_sensor_dims(&mut self, context: &HostContext<'_>, frame: &PluginFrame<'_>) { + self.sensor_dims = context + .get::(CTX_GLOBAL_SETTINGS) + .ok() + .flatten() + .map(|settings| (settings.sensor_width, settings.sensor_height)) + .or(Some((frame.width(), frame.height()))); + } + + fn analyze_candidates( + &mut self, + candidates: Option<&EveCandidates>, + output: &mut HostOutput<'_>, + nm_per_pixel: f64, + ) -> ( + EveLocalizationResults, + LocalizationResults, + Vec, + ) { + let Some(candidates) = candidates else { + self.last_localization_count = 0; + self.last_rejection_count = 0; + self.last_fit_failure_count = 0; + self.last_sigma_rejection_count = 0; + self.last_residual_rejection_count = 0; + self.last_status = "Waiting for EVE Candidate Finding.".into(); + Self::warning( + output, + AnalysisSeverity::Info, + "EVE fitting requires candidate clusters from EVE Candidate Finding.", + ); + return ( + EveLocalizationResults::default(), + LocalizationResults::default(), + Vec::new(), + ); + }; + + let mut localizations = Vec::new(); + let mut rejected_fits = Vec::new(); + let mut fit_failures = 0usize; + let mut sigma_rejections = 0usize; + let mut residual_rejections = 0usize; + for cluster in &candidates.clusters { + let (span_start_us, span_end_us) = cluster_time_span(cluster); + let timestamp_fallback = estimate_timestamp_us( + &cluster.events, + cluster.centroid_x, + cluster.centroid_y, + cluster_extent_radius(cluster), + ); + let Some(fit) = fit_cluster(cluster, self.settings.fit_method) else { + fit_failures += 1; + let mut rejected = RejectedFitRow { + row_id: 0, + cluster_id: cluster.cluster_id, + x: cluster.centroid_x, + y: cluster.centroid_y, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.0, + n_events: cluster.event_count() as u64, + polarity_balance: cluster.polarity_balance(), + rejection_reason: RejectionReason::FitFailed, + timestamp_us: timestamp_fallback, + span_start_us, + span_end_us, + }; + rejected.row_id = rejected_fit_row_id(&rejected); + rejected_fits.push(rejected); + continue; + }; + + if self.settings.fit_method.produces_sigma() { + let sigma_x_nm = fit.sigma_x * nm_per_pixel; + let sigma_y_nm = fit.sigma_y * nm_per_pixel; + if sigma_x_nm < self.settings.sigma_min_nm + || sigma_x_nm > self.settings.sigma_max_nm + || sigma_y_nm < self.settings.sigma_min_nm + || sigma_y_nm > self.settings.sigma_max_nm + { + sigma_rejections += 1; + let timestamp_us = estimate_timestamp_us( + &cluster.events, + fit.x, + fit.y, + fit_radius(cluster, &fit), + ); + let mut rejected = RejectedFitRow { + row_id: 0, + cluster_id: cluster.cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + fit_residual: fit.residual, + n_events: cluster.event_count() as u64, + polarity_balance: cluster.polarity_balance(), + rejection_reason: RejectionReason::SigmaOutOfBounds, + timestamp_us, + span_start_us, + span_end_us, + }; + rejected.row_id = rejected_fit_row_id(&rejected); + rejected_fits.push(rejected); + continue; + } + } + + if fit.residual > self.settings.max_fit_residual { + residual_rejections += 1; + let timestamp_us = + estimate_timestamp_us(&cluster.events, fit.x, fit.y, fit_radius(cluster, &fit)); + let mut rejected = RejectedFitRow { + row_id: 0, + cluster_id: cluster.cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + fit_residual: fit.residual, + n_events: cluster.event_count() as u64, + polarity_balance: cluster.polarity_balance(), + rejection_reason: RejectionReason::ResidualTooHigh, + timestamp_us, + span_start_us, + span_end_us, + }; + rejected.row_id = rejected_fit_row_id(&rejected); + rejected_fits.push(rejected); + continue; + } + + localizations.push(EveLocalization { + cluster_id: cluster.cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + timestamp_us: estimate_timestamp_us( + &cluster.events, + fit.x, + fit.y, + fit_radius(cluster, &fit), + ), + span_start_us, + span_end_us, + n_events: cluster.event_count(), + polarity_balance: cluster.polarity_balance(), + fit_residual: fit.residual, + fit_method: self.settings.fit_method, + }); + } + + self.last_localization_count = localizations.len(); + self.last_fit_failure_count = fit_failures; + self.last_sigma_rejection_count = sigma_rejections; + self.last_residual_rejection_count = residual_rejections; + self.last_rejection_count = fit_failures + sigma_rejections + residual_rejections; + self.last_status = format!( + "{} accepted, {} rejected ({} fit failures, {} sigma bounds, {} residual) with {}.", + self.last_localization_count, + self.last_rejection_count, + self.last_fit_failure_count, + self.last_sigma_rejection_count, + self.last_residual_rejection_count, + self.settings.fit_method.label() + ); + + if self.settings.show_overlay && !localizations.is_empty() { + let stable_ids: Vec = localizations + .iter() + .map(|localization| localization_row_id(localization).to_string()) + .collect(); + let markers: Vec = localizations + .iter() + .zip(stable_ids.iter()) + .map(|(localization, stable_id)| FfiMarkerOverlayItem { + x: localization.x as f32, + y: localization.y as f32, + shape: FfiMarkerShape::Cross, + size: 6.0, + color: FfiColorRgba::from_rgba(OVERLAY_COLOR), + timestamp_us: localization.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(CURRENT_LOCALIZATIONS_DATASET_ID), + Some(CURRENT_LOCALIZATIONS_LAYER_ID), + Some(self.name()), + ); + } + + if self.settings.show_rejected_overlay && !rejected_fits.is_empty() { + let stable_ids: Vec = rejected_fits + .iter() + .map(|row| row.row_id.to_string()) + .collect(); + let markers: Vec = rejected_fits + .iter() + .zip(stable_ids.iter()) + .map(|(row, stable_id)| FfiMarkerOverlayItem { + x: row.x as f32, + y: row.y as f32, + shape: FfiMarkerShape::Diamond, + size: 5.0, + color: FfiColorRgba::from_rgba([255, 90, 90, 180]), + timestamp_us: row.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: REJECTED_FITS_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(REJECTED_FITS_DATASET_ID), + Some(REJECTED_FITS_LAYER_ID), + Some(self.name()), + ); + } + + let eve_results = EveLocalizationResults { + localizations, + frame_window_start_us: candidates.frame_window_start_us, + frame_window_end_us: candidates.frame_window_end_us, + }; + let compatibility_results = to_localization_results(&eve_results); + + (eve_results, compatibility_results, rejected_fits) + } + + pub fn reset(&mut self) { + self.current_results = EveLocalizationResults::default(); + self.current_rejected_fits.clear(); + self.host_results = EveLocalizationResults::default(); + self.host_rejected_fits.clear(); + self.sensor_dims = None; + self.last_localization_count = 0; + self.last_rejection_count = 0; + self.last_fit_failure_count = 0; + self.last_sigma_rejection_count = 0; + self.last_residual_rejection_count = 0; + self.last_status = "Waiting for the next candidate set.".into(); + self.refit_preview_results = EveLocalizationResults::default(); + self.refit_preview_replaces.clear(); + self.last_action_notice = None; + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn parse_usize(value: Value) -> Option { + value.as_u64().and_then(|value| usize::try_from(value).ok()) + } + + fn update_history_bounds(results: &mut EveLocalizationResults) { + let Some(first) = results.localizations.first() else { + results.frame_window_start_us = 0; + results.frame_window_end_us = 0; + return; + }; + let mut start = first.span_start_us; + let mut end = first.span_end_us.max(first.span_start_us); + for localization in &results.localizations[1..] { + start = start.min(localization.span_start_us); + end = end.max(localization.span_end_us.max(localization.span_start_us)); + } + results.frame_window_start_us = start; + results.frame_window_end_us = end; + } + + fn upsert_history_localization(&mut self, localization: EveLocalization) { + self.host_rejected_fits + .retain(|row| row.cluster_id != localization.cluster_id); + if let Some(index) = self + .host_results + .localizations + .iter() + .position(|existing| existing.cluster_id == localization.cluster_id) + { + self.host_results.localizations[index] = localization; + } else { + self.host_results.localizations.push(localization); + } + self.host_results.localizations.sort_by_key(|row| { + ( + row.span_start_us, + row.span_end_us, + row.timestamp_us, + row.cluster_id, + ) + }); + Self::update_history_bounds(&mut self.host_results); + } + + fn upsert_history_rejected_fit(&mut self, row: RejectedFitRow) { + if self + .host_results + .localizations + .iter() + .any(|localization| localization.cluster_id == row.cluster_id) + { + return; + } + if let Some(index) = self + .host_rejected_fits + .iter() + .position(|existing| existing.cluster_id == row.cluster_id) + { + self.host_rejected_fits[index] = row; + } else { + self.host_rejected_fits.push(row); + } + self.host_rejected_fits.sort_by_key(|entry| { + ( + entry.span_start_us, + entry.span_end_us, + entry.timestamp_us, + entry.cluster_id, + ) + }); + } + + fn integrate_frame_history( + &mut self, + localizations: &[EveLocalization], + rejected_fits: &[RejectedFitRow], + ) { + for localization in localizations.iter().cloned() { + self.upsert_history_localization(localization); + } + for row in rejected_fits.iter().cloned() { + self.upsert_history_rejected_fit(row); + } + } + + fn parse_u64_field(value: &Value) -> Option { + value + .as_u64() + .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + } + + fn parse_u16_field(value: &Value) -> Option { + Self::parse_u64_field(value) + .and_then(|value| u16::try_from(value).ok()) + .or_else(|| { + value + .as_f64() + .map(|value| value.round().clamp(0.0, f64::from(u16::MAX)) as u16) + }) + } + + fn parse_bool_field(value: &Value) -> Option { + value + .as_bool() + .or_else(|| value.as_str().and_then(|value| value.parse().ok())) + } + + fn cluster_from_action_params(params: &Value, expected_cluster_id: u64) -> Option { + let rows = params.get(HOST_ACTION_CLUSTER_ROWS_PARAM)?.as_array()?; + if rows.is_empty() { + return None; + } + + let mut events = Vec::with_capacity(rows.len()); + let mut pixel_histogram: HashMap<(u16, u16), (u32, u32)> = HashMap::new(); + let mut sum_x = 0.0; + let mut sum_y = 0.0; + let mut count: f64 = 0.0; + let mut x_min = u16::MAX; + let mut x_max = 0u16; + let mut y_min = u16::MAX; + let mut y_max = 0u16; + + for row in rows { + let object = row.as_object()?; + let cluster_id = Self::parse_u64_field(object.get("cluster_id")?)?; + if cluster_id != expected_cluster_id { + return None; + } + let x = Self::parse_u16_field(object.get("x_px")?)?; + let y = Self::parse_u16_field(object.get("y_px")?)?; + let timestamp = Self::parse_u64_field(object.get("timestamp_us")?)?; + let polarity = Self::parse_bool_field(object.get("polarity")?)?; + + events.push(EveEvent { + timestamp, + x, + y, + polarity, + }); + + let entry = pixel_histogram.entry((x, y)).or_insert((0, 0)); + if polarity { + entry.0 = entry.0.saturating_add(1); + } else { + entry.1 = entry.1.saturating_add(1); + } + x_min = x_min.min(x); + x_max = x_max.max(x); + y_min = y_min.min(y); + y_max = y_max.max(y); + sum_x += f64::from(x); + sum_y += f64::from(y); + count += 1.0; + } + + if events.is_empty() { + return None; + } + + let mut pixel_histogram: Vec<_> = pixel_histogram + .into_iter() + .map(|((x, y), (positive, negative))| (x, y, positive, negative)) + .collect(); + pixel_histogram.sort_by_key(|(x, y, _, _)| (*y, *x)); + + Some(EveCluster { + cluster_id: expected_cluster_id, + pixel_histogram, + events, + centroid_x: sum_x / count.max(1.0), + centroid_y: sum_y / count.max(1.0), + x_min, + x_max, + y_min, + y_max, + complete: true, + boundary: None, + }) + } + + fn warning(output: &mut HostOutput<'_>, severity: AnalysisSeverity, message: &str) { + output.add_warning("EVE Candidate Fitting", severity, message); + } + + fn handle_action_requests( + &mut self, + context: &mut HostContext<'_>, + output: &mut HostOutput<'_>, + candidates: Option<&EveCandidates>, + nm_per_pixel: f64, + ) { + let queue = match context + .get_persistent::(CTX_INVESTIGATION_ACTION_REQUESTS) + { + Ok(Some(queue)) => queue, + Ok(None) => return, + Err(err) => { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Reading action requests failed: {err}"), + ); + return; + } + }; + + let mut handled_any = false; + for request in &queue.requests { + if request.request_id <= self.last_consumed_action_request_id { + continue; + } + match request.action_id.as_str() { + ACTION_REFIT_CLUSTER => { + self.handle_refit_cluster(request, output, candidates, nm_per_pixel); + handled_any = true; + } + ACTION_COMMIT_REFIT => { + self.handle_commit_refit(request, output); + handled_any = true; + } + ACTION_DISCARD_REFIT => { + self.handle_discard_refit(request, output); + handled_any = true; + } + _ => continue, + } + self.last_consumed_action_request_id = request.request_id; + } + + if handled_any { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + } + + fn handle_refit_cluster( + &mut self, + request: &augur_plugin_api::HostActionRequest, + output: &mut HostOutput<'_>, + candidates: Option<&EveCandidates>, + nm_per_pixel: f64, + ) { + use augur_plugin_api::HostActionScopePayload; + let (dataset_id, group_column, group_value) = match &request.scope_payload { + HostActionScopePayload::Cluster { + dataset_id, + group_column, + group_value, + } => ( + dataset_id.clone(), + group_column.clone(), + group_value.clone(), + ), + _ => { + Self::warning( + output, + AnalysisSeverity::Warning, + "Re-fit action requires a Cluster scope payload.", + ); + return; + } + }; + if dataset_id != ACCEPTED_CANDIDATE_EVENTS_DATASET_ID || group_column != "cluster_id" { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!( + "Ignoring re-fit request for unsupported scope ({dataset_id}/{group_column})." + ), + ); + return; + } + + let cluster_id: u64 = match group_value.parse() { + Ok(value) => value, + Err(_) => { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Re-fit request has non-numeric cluster id: {group_value}"), + ); + return; + } + }; + + let params = &request.params; + let cluster_from_params = Self::cluster_from_action_params(params, cluster_id); + let cluster_from_candidates = candidates.and_then(|candidates| { + candidates + .clusters + .iter() + .find(|cluster| cluster.cluster_id == cluster_id) + .cloned() + }); + let Some(cluster) = cluster_from_params.or(cluster_from_candidates) else { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Re-fit request for cluster {cluster_id} has no usable cluster snapshot."), + ); + return; + }; + let fit_method = params + .get("fit_method") + .and_then(|value| Self::parse_usize(value.clone())) + .map(FitMethod::from_index) + .unwrap_or(self.settings.fit_method); + let sigma_min_nm = params + .get("sigma_min_nm") + .and_then(Value::as_f64) + .unwrap_or(self.settings.sigma_min_nm); + let sigma_max_nm = params + .get("sigma_max_nm") + .and_then(Value::as_f64) + .unwrap_or(self.settings.sigma_max_nm); + let max_fit_residual = params + .get("max_fit_residual") + .and_then(Value::as_f64) + .unwrap_or(self.settings.max_fit_residual); + + let Some(fit) = fit_cluster(&cluster, fit_method) else { + self.last_action_notice = Some(format!("Re-fit failed for cluster {cluster_id}.")); + Self::warning( + output, + AnalysisSeverity::Info, + &format!("Re-fit for cluster {cluster_id} did not converge."), + ); + return; + }; + + if fit_method.produces_sigma() { + let sigma_x_nm = fit.sigma_x * nm_per_pixel; + let sigma_y_nm = fit.sigma_y * nm_per_pixel; + if sigma_x_nm < sigma_min_nm + || sigma_x_nm > sigma_max_nm + || sigma_y_nm < sigma_min_nm + || sigma_y_nm > sigma_max_nm + { + self.last_action_notice = Some(format!( + "Re-fit for cluster {cluster_id} is outside sigma bounds." + )); + Self::warning( + output, + AnalysisSeverity::Info, + &format!("Re-fit for cluster {cluster_id} rejected by sigma bounds."), + ); + return; + } + } + + if fit.residual > max_fit_residual { + self.last_action_notice = Some(format!( + "Re-fit for cluster {cluster_id} exceeds residual threshold." + )); + Self::warning( + output, + AnalysisSeverity::Info, + &format!("Re-fit for cluster {cluster_id} rejected by residual threshold."), + ); + return; + } + + let timestamp_us = + estimate_timestamp_us(&cluster.events, fit.x, fit.y, fit_radius(&cluster, &fit)); + let (span_start_us, span_end_us) = cluster_time_span(&cluster); + let new_localization = EveLocalization { + cluster_id, + x: fit.x, + y: fit.y, + sigma_x: fit.sigma_x, + sigma_y: fit.sigma_y, + timestamp_us, + span_start_us, + span_end_us, + n_events: cluster.event_count(), + polarity_balance: cluster.polarity_balance(), + fit_residual: fit.residual, + fit_method, + }; + + let replaces = find_current_localization_for_cluster(&self.host_results, &cluster) + .map(localization_row_id); + + self.refit_preview_results + .localizations + .push(new_localization); + self.refit_preview_replaces.push(replaces); + Self::update_history_bounds(&mut self.refit_preview_results); + + self.last_action_notice = Some(format!( + "Re-fit preview added for cluster {cluster_id} ({}).", + fit_method.label() + )); + } + + fn handle_commit_refit( + &mut self, + request: &augur_plugin_api::HostActionRequest, + output: &mut HostOutput<'_>, + ) { + use augur_plugin_api::HostActionScopePayload; + let (dataset_id, row_id) = match &request.scope_payload { + HostActionScopePayload::Row { dataset_id, row_id } => { + (dataset_id.clone(), row_id.clone()) + } + _ => { + Self::warning( + output, + AnalysisSeverity::Warning, + "Commit action requires a Row scope payload.", + ); + return; + } + }; + if dataset_id != REFIT_PREVIEW_DATASET_ID { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Ignoring commit for unsupported dataset {dataset_id}."), + ); + return; + } + + let target_row_id: u64 = match row_id.parse() { + Ok(value) => value, + Err(_) => { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Commit row_id is not numeric: {row_id}"), + ); + return; + } + }; - fn analyze_candidates( - &mut self, - candidates: Option<&EveCandidates>, - output: &mut HostOutput<'_>, - nm_per_pixel: f64, - ) -> (EveLocalizationResults, LocalizationResults) { - let Some(candidates) = candidates else { - self.last_localization_count = 0; - self.last_rejection_count = 0; - self.last_status = "Waiting for EVE Candidate Finding.".into(); + let index = self + .refit_preview_results + .localizations + .iter() + .position(|localization| localization_row_id(localization) == target_row_id); + let Some(index) = index else { Self::warning( output, AnalysisSeverity::Info, - "EVE fitting requires candidate clusters from EVE Candidate Finding.", - ); - return ( - EveLocalizationResults::default(), - LocalizationResults::default(), + &format!("Commit row {target_row_id} is not in the preview."), ); + return; }; - let mut localizations = Vec::new(); - let mut rejected = 0; - for cluster in &candidates.clusters { - let Some(fit) = fit_cluster(cluster, self.settings.fit_method) else { - rejected += 1; - continue; - }; - - if self.settings.fit_method.produces_sigma() { - let sigma_x_nm = fit.sigma_x * nm_per_pixel; - let sigma_y_nm = fit.sigma_y * nm_per_pixel; - if sigma_x_nm < self.settings.sigma_min_nm - || sigma_x_nm > self.settings.sigma_max_nm - || sigma_y_nm < self.settings.sigma_min_nm - || sigma_y_nm > self.settings.sigma_max_nm - { - rejected += 1; - continue; - } - } - - if fit.residual > self.settings.max_fit_residual { - rejected += 1; - continue; - } + let localization = self.refit_preview_results.localizations.remove(index); + self.refit_preview_replaces.remove(index); + let cluster_id = localization.cluster_id; + self.upsert_history_localization(localization.clone()); + self.host_rejected_fits + .retain(|row| row.cluster_id != cluster_id); - localizations.push(EveLocalization { - x: fit.x, - y: fit.y, - sigma_x: fit.sigma_x, - sigma_y: fit.sigma_y, - timestamp_us: estimate_timestamp_us( - &cluster.events, - fit.x, - fit.y, - fit_radius(cluster, &fit), - ), - n_events: cluster.event_count(), - polarity_balance: cluster.polarity_balance(), - fit_residual: fit.residual, - fit_method: self.settings.fit_method, - }); + if let Some(old_index) = self + .current_results + .localizations + .iter() + .position(|entry| entry.cluster_id == cluster_id) + { + self.current_results.localizations[old_index] = localization; } - self.last_localization_count = localizations.len(); - self.last_rejection_count = rejected; - self.last_status = format!( - "{} localizations accepted, {} rejected with {}.", - self.last_localization_count, - self.last_rejection_count, - self.settings.fit_method.label() - ); + Self::update_history_bounds(&mut self.refit_preview_results); - if self.settings.show_overlay && !localizations.is_empty() { - let markers: Vec = localizations - .iter() - .map(|localization| FfiSubpixelMarker { - x: localization.x as f32, - y: localization.y as f32, - }) - .collect(); - output.add_crosshair_markers(&markers, OVERLAY_COLOR, 5); - } + self.last_action_notice = Some(format!("Committed refit preview row {target_row_id}.")); + } - let eve_results = EveLocalizationResults { - localizations, - frame_window_start_us: candidates.frame_window_start_us, - frame_window_end_us: candidates.frame_window_end_us, + fn handle_discard_refit( + &mut self, + request: &augur_plugin_api::HostActionRequest, + output: &mut HostOutput<'_>, + ) { + use augur_plugin_api::HostActionScopePayload; + let dataset_id = match &request.scope_payload { + HostActionScopePayload::Dataset { dataset_id } => dataset_id.clone(), + _ => { + Self::warning( + output, + AnalysisSeverity::Warning, + "Discard action requires a Dataset scope payload.", + ); + return; + } }; - let compatibility_results = to_localization_results(&eve_results); + if dataset_id != REFIT_PREVIEW_DATASET_ID { + Self::warning( + output, + AnalysisSeverity::Warning, + &format!("Ignoring discard for unsupported dataset {dataset_id}."), + ); + return; + } - (eve_results, compatibility_results) + let dropped = self.refit_preview_results.localizations.len(); + self.refit_preview_results = EveLocalizationResults::default(); + self.refit_preview_replaces.clear(); + self.last_action_notice = Some(format!("Discarded {dropped} preview row(s).")); } - pub fn reset(&mut self) { - self.current_results = EveLocalizationResults::default(); - self.last_localization_count = 0; - self.last_rejection_count = 0; - self.last_status = "Waiting for the next candidate set.".into(); - self.dataset_generation = self.dataset_generation.wrapping_add(1); + fn emit_refit_preview_overlay(&self, output: &mut HostOutput<'_>) { + let localizations = &self.refit_preview_results.localizations; + let stable_ids: Vec = localizations + .iter() + .map(|localization| localization_row_id(localization).to_string()) + .collect(); + let markers: Vec = localizations + .iter() + .zip(stable_ids.iter()) + .map(|(localization, stable_id)| FfiMarkerOverlayItem { + x: localization.x as f32, + y: localization.y as f32, + shape: FfiMarkerShape::FilledCircle, + size: 8.0, + color: FfiColorRgba::from_rgba([255, 210, 90, 240]), + timestamp_us: localization.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), + }) + .collect(); + output.add_marker_overlay( + &markers, + Some(REFIT_PREVIEW_DATASET_ID), + Some(REFIT_PREVIEW_LAYER_ID), + Some(self.name()), + ); } +} - fn parse_usize(value: Value) -> Option { - value.as_u64().and_then(|value| usize::try_from(value).ok()) +fn find_current_localization_for_cluster<'a>( + results: &'a EveLocalizationResults, + cluster: &EveCluster, +) -> Option<&'a EveLocalization> { + if let Some(localization) = results + .localizations + .iter() + .find(|localization| localization.cluster_id == cluster.cluster_id) + { + return Some(localization); + } + let timestamp_range_us: i64 = 2_000; + let mut best: Option<(f64, &'a EveLocalization)> = None; + for localization in &results.localizations { + let dt = (localization.timestamp_us as i64) + .saturating_sub_unsigned(cluster_anchor_timestamp(cluster)); + if dt.abs() > timestamp_range_us { + continue; + } + let dx = localization.x - cluster.centroid_x; + let dy = localization.y - cluster.centroid_y; + let score = dx * dx + dy * dy + (dt as f64).powi(2) * 1e-6; + if best.map_or(true, |(b, _)| score < b) { + best = Some((score, localization)); + } } + best.map(|(_, localization)| localization) +} - fn warning(output: &mut HostOutput<'_>, severity: AnalysisSeverity, message: &str) { - output.add_warning("EVE Candidate Fitting", severity, message); +fn cluster_anchor_timestamp(cluster: &EveCluster) -> u64 { + if cluster.events.is_empty() { + return 0; } + let sum: u128 = cluster + .events + .iter() + .map(|event| event.timestamp as u128) + .sum(); + (sum / cluster.events.len() as u128) as u64 } impl Plugin for EveSmlmFittingPlugin { @@ -344,11 +1953,12 @@ impl Plugin for EveSmlmFittingPlugin { fn process_frame( &mut self, - _frame: &PluginFrame<'_>, + frame: &PluginFrame<'_>, output: &mut HostOutput<'_>, context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { + self.sync_sensor_dims(context, frame); let nm_per_pixel = self.nm_per_pixel(context); let candidates = match context.get::(CTX_EVE_CANDIDATES) { Ok(value) => value, @@ -362,11 +1972,22 @@ impl Plugin for EveSmlmFittingPlugin { } }; - let (eve_results, compatibility) = + let (eve_results, _compatibility, rejected_fits) = self.analyze_candidates(candidates.as_ref(), output, nm_per_pixel); self.current_results = eve_results.clone(); + self.current_rejected_fits = rejected_fits.clone(); + self.integrate_frame_history(&eve_results.localizations, &rejected_fits); self.dataset_generation = self.dataset_generation.wrapping_add(1); - if let Err(err) = context.publish(CTX_EVE_LOCALIZATION_RESULTS, &eve_results) { + + self.handle_action_requests(context, output, candidates.as_ref(), nm_per_pixel); + + if self.settings.show_overlay && !self.refit_preview_results.localizations.is_empty() { + self.emit_refit_preview_overlay(output); + } + + let published_results = self.current_results.clone(); + let compatibility = to_localization_results(&published_results); + if let Err(err) = context.publish(CTX_EVE_LOCALIZATION_RESULTS, &published_results) { Self::warning( output, AnalysisSeverity::Warning, @@ -462,6 +2083,16 @@ impl Plugin for EveSmlmFittingPlugin { default: self.settings.show_overlay, }, }, + SettingItem { + key: "show_rejected_overlay".into(), + label: "Show rejected".into(), + tooltip: Some( + "Draw rejected fits as linked diamond markers in the preview.".into(), + ), + kind: SettingKind::Bool { + default: self.settings.show_rejected_overlay, + }, + }, ], }], } @@ -474,6 +2105,7 @@ impl Plugin for EveSmlmFittingPlugin { "sigma_max_nm" => Some(json!(self.settings.sigma_max_nm)), "max_fit_residual" => Some(json!(self.settings.max_fit_residual)), "show_overlay" => Some(json!(self.settings.show_overlay)), + "show_rejected_overlay" => Some(json!(self.settings.show_rejected_overlay)), _ => None, } } @@ -520,6 +2152,12 @@ impl Plugin for EveSmlmFittingPlugin { }; self.settings.show_overlay = value; } + "show_rejected_overlay" => { + let Some(value) = value.as_bool() else { + return Err("show_rejected_overlay must be a boolean".into()); + }; + self.settings.show_rejected_overlay = value; + } _ => return Err(format!("unknown setting: {key}")), } @@ -527,7 +2165,7 @@ impl Plugin for EveSmlmFittingPlugin { } fn status_entries(&self) -> Vec { - vec![ + let mut entries = vec![ StatusEntry::Text(self.last_status.clone()), StatusEntry::LabeledValue { label: "Accepted".into(), @@ -544,26 +2182,95 @@ impl Plugin for EveSmlmFittingPlugin { value: self.settings.fit_method.label().into(), color: None, }, - ] + ]; + if self.last_rejection_count > 0 { + entries.push(StatusEntry::LabeledValue { + label: "Fit fail".into(), + value: self.last_fit_failure_count.to_string(), + color: None, + }); + entries.push(StatusEntry::LabeledValue { + label: "Sigma".into(), + value: self.last_sigma_rejection_count.to_string(), + color: None, + }); + entries.push(StatusEntry::LabeledValue { + label: "Residual".into(), + value: self.last_residual_rejection_count.to_string(), + color: None, + }); + } + entries } fn host_views(&self) -> HostViewRegistry { - current_localizations_registry() + let mut registry = + current_localizations_registry_for_results(&self.host_results, self.sensor_dims); + let rejected_registry = rejected_fits_registry( + &self.host_rejected_fits, + self.sensor_dims, + self.host_results.frame_window_start_us, + self.host_results.frame_window_end_us, + ); + registry.datasets.extend(rejected_registry.datasets); + registry.views.extend(rejected_registry.views); + let preview_registry = + refit_preview_registry_for_results(&self.refit_preview_results, self.sensor_dims); + registry.datasets.extend(preview_registry.datasets); + registry.views.extend(preview_registry.views); + + let param_schema = serde_json::to_value(refit_action_param_schema()).ok(); + registry.actions = vec![ + HostActionDescriptor { + id: ACTION_REFIT_CLUSTER.into(), + title: "Re-fit cluster…".into(), + scope: HostActionScope::Cluster { + dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + group_column: "cluster_id".into(), + }, + param_schema, + }, + HostActionDescriptor { + id: ACTION_COMMIT_REFIT.into(), + title: "Commit refit".into(), + scope: HostActionScope::Row { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + }, + param_schema: None, + }, + HostActionDescriptor { + id: ACTION_DISCARD_REFIT.into(), + title: "Discard refit preview".into(), + scope: HostActionScope::Dataset { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + }, + param_schema: None, + }, + ]; + registry } fn host_view_dataset(&self, dataset_id: &str) -> Option> { - if dataset_id != CURRENT_LOCALIZATIONS_DATASET_ID { - return None; + match dataset_id { + CURRENT_LOCALIZATIONS_DATASET_ID => { + serde_json::to_vec(¤t_localizations_dataset(&self.host_results)).ok() + } + REJECTED_FITS_DATASET_ID => { + serde_json::to_vec(&rejected_fits_dataset(&self.host_rejected_fits)).ok() + } + REFIT_PREVIEW_DATASET_ID => { + serde_json::to_vec(&refit_preview_dataset(&self.refit_preview_results)).ok() + } + _ => None, } - - serde_json::to_vec(¤t_localizations_dataset(&self.current_results)).ok() } fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - if dataset_id == CURRENT_LOCALIZATIONS_DATASET_ID { - self.dataset_generation - } else { - 0 + match dataset_id { + CURRENT_LOCALIZATIONS_DATASET_ID + | REJECTED_FITS_DATASET_ID + | REFIT_PREVIEW_DATASET_ID => self.dataset_generation, + _ => 0, } } } @@ -578,14 +2285,31 @@ fn fit_cluster(cluster: &EveCluster, method: FitMethod) -> Option { } } +fn cluster_extent_radius(cluster: &EveCluster) -> f64 { + let dx = f64::from(cluster.x_max.saturating_sub(cluster.x_min)) + 1.0; + let dy = f64::from(cluster.y_max.saturating_sub(cluster.y_min)) + 1.0; + 0.5 * dx.max(dy).max(1.0) +} + fn fit_radius(cluster: &EveCluster, fit: &FitEstimate) -> f64 { if fit.sigma_x > 0.0 && fit.sigma_y > 0.0 { 2.5 * fit.sigma_x.max(fit.sigma_y).max(1.0) } else { - let dx = f64::from(cluster.x_max.saturating_sub(cluster.x_min)) + 1.0; - let dy = f64::from(cluster.y_max.saturating_sub(cluster.y_min)) + 1.0; - 0.5 * dx.max(dy).max(1.0) + cluster_extent_radius(cluster) + } +} + +fn cluster_time_span(cluster: &EveCluster) -> (u64, u64) { + let Some(first) = cluster.events.first() else { + return (0, 0); + }; + let mut start = first.timestamp; + let mut end = first.timestamp; + for event in &cluster.events[1..] { + start = start.min(event.timestamp); + end = end.max(event.timestamp); } + (start, end.max(start)) } fn estimate_timestamp_us(events: &[EveEvent], x: f64, y: f64, radius: f64) -> u64 { @@ -654,6 +2378,23 @@ mod tests { } } + fn localization(x: f64, y: f64, timestamp_us: u64) -> EveLocalization { + EveLocalization { + cluster_id: timestamp_us, + x, + y, + sigma_x: 0.7, + sigma_y: 0.8, + timestamp_us, + span_start_us: timestamp_us.saturating_sub(1), + span_end_us: timestamp_us.saturating_add(1), + n_events: 7, + polarity_balance: 0.1, + fit_residual: 0.02, + fit_method: FitMethod::LogGaussian, + } + } + fn cluster_from_histogram(entries: &[(u16, u16, u32)]) -> EveCluster { let mut pixel_histogram = Vec::new(); let mut events = Vec::new(); @@ -682,6 +2423,7 @@ mod tests { } EveCluster { + cluster_id: 0, pixel_histogram, events, centroid_x: if total > 0.0 { sum_x / total } else { 0.0 }, @@ -690,6 +2432,8 @@ mod tests { x_max, y_min, y_max, + complete: true, + boundary: None, } } @@ -799,6 +2543,7 @@ mod tests { #[test] fn empty_cluster_returns_none() { let cluster = EveCluster { + cluster_id: 0, pixel_histogram: Vec::new(), events: Vec::new(), centroid_x: 0.0, @@ -807,6 +2552,8 @@ mod tests { x_max: 0, y_min: 0, y_max: 0, + complete: true, + boundary: None, }; assert!(mean_xy::fit(&cluster).is_none()); @@ -819,33 +2566,442 @@ mod tests { let registry = current_localizations_registry(); assert_eq!(registry.datasets.len(), 1); - assert_eq!(registry.views.len(), 1); + assert_eq!(registry.views.len(), 2); assert_eq!(registry.datasets[0].id, CURRENT_LOCALIZATIONS_DATASET_ID); assert_eq!(registry.views[0].id, CURRENT_LOCALIZATIONS_VIEW_ID); + assert_eq!(registry.views[1].id, CURRENT_LOCALIZATIONS_3D_VIEW_ID); + assert!(registry.datasets[0].display.is_some()); } #[test] fn host_view_dataset_is_columnar_and_aligned() { let dataset = current_localizations_dataset(&EveLocalizationResults { - localizations: vec![EveLocalization { - x: 1.5, - y: 2.5, - sigma_x: 0.7, - sigma_y: 0.8, - timestamp_us: 10, - n_events: 7, - polarity_balance: 0.1, - fit_residual: 0.02, - fit_method: FitMethod::LogGaussian, - }], + localizations: vec![localization(1.5, 2.5, 10)], + frame_window_start_us: 0, + frame_window_end_us: 50, + }); + + assert_eq!(dataset.row_count(), 1); + assert_eq!(dataset.columns.len(), 13); + assert_eq!(dataset.columns[0].column_id, "row_id"); + assert_eq!(dataset.columns[1].column_id, "cluster_id"); + assert_eq!(dataset.columns[4].column_id, "span_end_us"); + assert_eq!(dataset.columns[12].column_id, "fit_method"); + } + + #[test] + fn current_localization_schema_exposes_linking_metadata() { + let schema = current_localizations_schema_for_results( + &EveLocalizationResults { + localizations: vec![localization(12.0, 18.0, 15)], + frame_window_start_us: 10, + frame_window_end_us: 20, + }, + Some((128, 64)), + ); + assert_eq!(schema.row_id_column.as_deref(), Some("row_id")); + assert_eq!(schema.time_column.as_deref(), Some("timestamp_us")); + assert_eq!( + schema + .coordinate_space_3d + .as_ref() + .map(|space| space.z_column.as_str()), + Some("timestamp_us") + ); + assert_eq!( + schema.layer_id.as_deref(), + Some(CURRENT_LOCALIZATIONS_LAYER_ID) + ); + let provenance = schema.provenance.as_ref().expect("provenance"); + assert_eq!( + provenance.anchor_time_column.as_deref(), + Some("timestamp_us") + ); + assert_eq!( + provenance.span_start_column.as_deref(), + Some("span_start_us") + ); + assert_eq!(provenance.span_end_column.as_deref(), Some("span_end_us")); + } + + #[test] + fn current_localization_registry_relates_rows_to_accepted_candidate_events() { + let registry = current_localizations_registry_for_results( + &EveLocalizationResults { + localizations: vec![localization(12.0, 18.0, 15)], + frame_window_start_us: 10, + frame_window_end_us: 20, + }, + Some((128, 64)), + ); + let relations = ®istry.datasets[0].relations; + assert_eq!(relations.len(), 1); + assert_eq!( + relations[0].target_dataset_id, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID + ); + assert_eq!(relations[0].via_column, "cluster_id"); + assert_eq!(relations[0].target_column, "cluster_id"); + } + + #[test] + fn current_localization_dataset_uses_repeatable_row_ids() { + let dataset = current_localizations_dataset(&EveLocalizationResults { + localizations: vec![localization(1.0, 2.0, 11), localization(1.0, 2.0, 11)], frame_window_start_us: 0, frame_window_end_us: 50, }); + let ids = match &dataset.column("row_id").expect("row id column").values { + TableColumnValues::U64(values) => values.clone(), + other => panic!("unexpected row id values: {other:?}"), + }; + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[test] + fn rejected_fit_registry_exposes_dataset_table_and_3d_views() { + let registry = rejected_fits_registry( + &[RejectedFitRow { + row_id: 1, + cluster_id: 7, + x: 10.5, + y: 12.5, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.0, + n_events: 5, + polarity_balance: 0.2, + rejection_reason: RejectionReason::FitFailed, + timestamp_us: 15, + span_start_us: 10, + span_end_us: 20, + }], + Some((128, 64)), + 10, + 20, + ); + + assert_eq!(registry.datasets.len(), 1); + assert_eq!(registry.views.len(), 3); + assert_eq!(registry.datasets[0].id, REJECTED_FITS_DATASET_ID); + assert_eq!(registry.views[0].id, REJECTED_FITS_COMPACT_VIEW_ID); + assert!(matches!(registry.views[0].kind, HostViewKind::CompactTable)); + assert_eq!(registry.views[1].id, REJECTED_FITS_TABLE_VIEW_ID); + assert!(matches!(registry.views[1].kind, HostViewKind::TableWindow)); + assert_eq!(registry.views[2].id, REJECTED_FITS_3D_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.row_id_column.as_deref(), Some("row_id")); + assert_eq!(schema.layer_id.as_deref(), Some(REJECTED_FITS_LAYER_ID)); + let provenance = schema.provenance.as_ref().expect("provenance"); + assert_eq!( + provenance.anchor_time_column.as_deref(), + Some("timestamp_us") + ); + assert_eq!( + provenance.span_start_column.as_deref(), + Some("span_start_us") + ); + assert_eq!(provenance.span_end_column.as_deref(), Some("span_end_us")); + assert_eq!(registry.datasets[0].relations.len(), 1); + assert_eq!( + registry.datasets[0].relations[0].target_dataset_id, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID + ); + } + + #[test] + fn rejected_fit_dataset_is_columnar_and_repeatable() { + let row = RejectedFitRow { + row_id: 99, + cluster_id: 5, + x: 4.0, + y: 6.0, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.1, + n_events: 8, + polarity_balance: -0.25, + rejection_reason: RejectionReason::ResidualTooHigh, + timestamp_us: 22, + span_start_us: 20, + span_end_us: 30, + }; + let dataset = rejected_fits_dataset(&[row.clone(), row]); + + assert_eq!(dataset.row_count(), 2); + assert_eq!(dataset.columns.len(), 13); + assert_eq!(dataset.columns[0].column_id, "row_id"); + assert_eq!(dataset.columns[12].column_id, "rejection_reason"); + } + + use std::ffi::c_void; + + use augur_plugin_api::{ + FfiColorRgba as TestFfiColorRgba, FfiMarkerOverlayItem as TestFfiMarkerOverlayItem, + FfiOutputCallbacks, FfiPixel, FfiSlice, FfiString, FfiSubpixelMarker, + }; + + unsafe extern "C" fn noop_pixels( + _ctx: *mut c_void, + _pixels: FfiSlice, + _color: TestFfiColorRgba, + ) { + } + unsafe extern "C" fn noop_crosshairs( + _ctx: *mut c_void, + _markers: FfiSlice, + _color: TestFfiColorRgba, + _arm: u16, + ) { + } + unsafe extern "C" fn noop_marker_overlay( + _ctx: *mut c_void, + _markers: FfiSlice, + _dataset: FfiString, + _layer: FfiString, + _src: FfiString, + ) { + } + unsafe extern "C" fn noop_warning( + _ctx: *mut c_void, + _source: FfiString, + _severity: AnalysisSeverity, + _message: FfiString, + ) { + } + + fn noop_output_callbacks() -> FfiOutputCallbacks { + FfiOutputCallbacks { + ctx: std::ptr::null_mut(), + add_highlight_pixels: noop_pixels, + add_crosshair_markers: noop_crosshairs, + add_marker_overlay: noop_marker_overlay, + add_warning: noop_warning, + } + } + + fn cluster_snapshot_params( + cluster_id: u64, + events: &[(u16, u16, bool, u64)], + fit_method: FitMethod, + ) -> Value { + let rows = events + .iter() + .map(|(x, y, polarity, timestamp_us)| { + json!({ + "cluster_id": cluster_id, + "x_px": x, + "y_px": y, + "polarity": polarity, + "timestamp_us": timestamp_us, + }) + }) + .collect(); + let mut params = serde_json::Map::new(); + params.insert("fit_method".into(), json!(fit_method.index() as u64)); + params.insert(HOST_ACTION_CLUSTER_ROWS_PARAM.into(), Value::Array(rows)); + Value::Object(params) + } + + #[test] + fn refit_preview_registry_uses_distinct_layer_and_dataset_ids() { + let registry = + refit_preview_registry_for_results(&EveLocalizationResults::default(), Some((64, 64))); + assert_eq!(registry.datasets.len(), 1); + assert_eq!(registry.datasets[0].id, REFIT_PREVIEW_DATASET_ID); + assert_eq!(registry.views.len(), 1); + assert_eq!(registry.views[0].id, REFIT_PREVIEW_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.layer_id.as_deref(), Some(REFIT_PREVIEW_LAYER_ID)); + assert_eq!(schema.semantic_label.as_deref(), Some("refit preview")); + } + + #[test] + fn host_views_registers_three_actions_with_expected_scopes() { + let plugin = EveSmlmFittingPlugin::default(); + let registry = plugin.host_views(); + + assert_eq!(registry.actions.len(), 3); + assert_eq!(registry.actions[0].id, ACTION_REFIT_CLUSTER); + assert!(matches!( + registry.actions[0].scope, + HostActionScope::Cluster { ref dataset_id, ref group_column } + if dataset_id == ACCEPTED_CANDIDATE_EVENTS_DATASET_ID + && group_column == "cluster_id" + )); + assert!(registry.actions[0].param_schema.is_some()); + + assert_eq!(registry.actions[1].id, ACTION_COMMIT_REFIT); + assert!(matches!( + registry.actions[1].scope, + HostActionScope::Row { ref dataset_id } if dataset_id == REFIT_PREVIEW_DATASET_ID + )); + assert!(registry.actions[1].param_schema.is_none()); + + assert_eq!(registry.actions[2].id, ACTION_DISCARD_REFIT); + assert!(matches!( + registry.actions[2].scope, + HostActionScope::Dataset { ref dataset_id } if dataset_id == REFIT_PREVIEW_DATASET_ID + )); + } + + #[test] + fn refit_cluster_uses_snapshot_rows_when_current_frame_cluster_is_missing() { + let mut plugin = EveSmlmFittingPlugin::default(); + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_REFIT_CLUSTER.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Cluster { + dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + group_column: "cluster_id".into(), + group_value: "7".into(), + }, + params: cluster_snapshot_params( + 7, + &[(10, 20, true, 100), (12, 20, false, 130)], + FitMethod::MeanXY, + ), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_refit_cluster(&request, &mut output, None, 65.0); + + assert_eq!(plugin.refit_preview_results.localizations.len(), 1); + let preview = &plugin.refit_preview_results.localizations[0]; + assert_eq!(preview.cluster_id, 7); + assert!((preview.x - 11.0).abs() < 1e-6); + assert!((preview.y - 20.0).abs() < 1e-6); + assert_eq!(preview.n_events, 2); + assert_eq!(preview.span_start_us, 100); + assert_eq!(preview.span_end_us, 130); + assert_eq!(plugin.refit_preview_replaces, vec![None]); + } + + #[test] + fn commit_persists_preview_into_host_results_even_without_current_frame_match() { + let mut plugin = EveSmlmFittingPlugin::default(); + let preview = localization(3.5, 4.5, 200); + let preview_row_id = localization_row_id(&preview); + plugin.refit_preview_results.localizations.push(preview); + plugin.refit_preview_replaces.push(None); + plugin.host_rejected_fits.push(RejectedFitRow { + row_id: 99, + cluster_id: 200, + x: 3.0, + y: 4.0, + sigma_x: 0.0, + sigma_y: 0.0, + fit_residual: 0.2, + n_events: 6, + polarity_balance: 0.1, + rejection_reason: RejectionReason::ResidualTooHigh, + timestamp_us: 180, + span_start_us: 170, + span_end_us: 210, + }); + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_COMMIT_REFIT.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Row { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + row_id: preview_row_id.to_string(), + }, + params: serde_json::json!({}), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_commit_refit(&request, &mut output); + + assert!(plugin.refit_preview_results.localizations.is_empty()); + assert!(plugin.current_results.localizations.is_empty()); + assert_eq!(plugin.host_results.localizations.len(), 1); + assert_eq!(plugin.host_results.localizations[0].cluster_id, 200); + assert_eq!(plugin.host_results.localizations[0].x, 3.5); + assert!(plugin.host_rejected_fits.is_empty()); + + let dataset_bytes = plugin + .host_view_dataset(CURRENT_LOCALIZATIONS_DATASET_ID) + .expect("host dataset bytes"); + let dataset: TableDatasetV1 = + serde_json::from_slice(&dataset_bytes).expect("table dataset should deserialize"); assert_eq!(dataset.row_count(), 1); - assert_eq!(dataset.columns.len(), 5); - assert_eq!(dataset.columns[0].column_id, "x_px"); - assert_eq!(dataset.columns[4].column_id, "n_events"); + } + + #[test] + fn commit_replaces_current_localization_when_cluster_matches_current_frame() { + let mut plugin = EveSmlmFittingPlugin::default(); + let old = localization(1.0, 2.0, 100); + plugin.current_results.localizations.push(old); + + let preview = localization(1.1, 2.1, 100); + let preview_row_id = localization_row_id(&preview); + plugin.refit_preview_results.localizations.push(preview); + plugin.refit_preview_replaces.push(None); + + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_COMMIT_REFIT.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Row { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + row_id: preview_row_id.to_string(), + }, + params: serde_json::json!({}), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_commit_refit(&request, &mut output); + + assert_eq!(plugin.current_results.localizations.len(), 1); + assert_eq!(plugin.current_results.localizations[0].x, 1.1); + assert_eq!(plugin.current_results.localizations[0].y, 2.1); + assert_eq!(plugin.host_results.localizations.len(), 1); + assert_eq!(plugin.host_results.localizations[0].cluster_id, 100); + } + + #[test] + fn discard_clears_preview_without_touching_current_results() { + let mut plugin = EveSmlmFittingPlugin::default(); + plugin + .current_results + .localizations + .push(localization(1.0, 2.0, 100)); + let baseline = plugin.current_results.clone(); + + plugin + .refit_preview_results + .localizations + .push(localization(9.0, 9.0, 900)); + plugin.refit_preview_replaces.push(None); + + let request = augur_plugin_api::HostActionRequest { + request_id: 1, + action_id: ACTION_DISCARD_REFIT.into(), + scope_payload: augur_plugin_api::HostActionScopePayload::Dataset { + dataset_id: REFIT_PREVIEW_DATASET_ID.into(), + }, + params: serde_json::json!({}), + }; + + let mut callbacks = noop_output_callbacks(); + let mut output = augur_plugin_api::HostOutput::new(&mut callbacks); + plugin.handle_discard_refit(&request, &mut output); + + assert!(plugin.refit_preview_results.localizations.is_empty()); + assert!(plugin.refit_preview_replaces.is_empty()); + let baseline_bytes = serde_json::to_vec(&baseline).unwrap(); + let after_bytes = serde_json::to_vec(&plugin.current_results).unwrap(); + assert_eq!(baseline_bytes, after_bytes); } } diff --git a/plugins/evesmlm-fitting/src/types.rs b/plugins/evesmlm-fitting/src/types.rs index 3c37295..85684bc 100644 --- a/plugins/evesmlm-fitting/src/types.rs +++ b/plugins/evesmlm-fitting/src/types.rs @@ -2,6 +2,32 @@ use serde::{Deserialize, Serialize}; pub const CTX_EVE_LOCALIZATION_RESULTS: &str = "augur.evesmlm.localization_results"; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RejectionReason { + FitFailed, + SigmaOutOfBounds, + ResidualTooHigh, +} + +impl RejectionReason { + pub fn label(self) -> &'static str { + match self { + Self::FitFailed => "Fit failed", + Self::SigmaOutOfBounds => "Sigma out of bounds", + Self::ResidualTooHigh => "Residual too high", + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::FitFailed => "fit_failed", + Self::SigmaOutOfBounds => "sigma_out_of_bounds", + Self::ResidualTooHigh => "residual_too_high", + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FitMethod { @@ -51,11 +77,14 @@ impl FitMethod { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct EveLocalization { + pub cluster_id: u64, pub x: f64, pub y: f64, pub sigma_x: f64, pub sigma_y: f64, pub timestamp_us: u64, + pub span_start_us: u64, + pub span_end_us: u64, pub n_events: usize, pub polarity_balance: f64, pub fit_residual: f64, @@ -68,3 +97,20 @@ pub struct EveLocalizationResults { pub frame_window_start_us: u64, pub frame_window_end_us: u64, } + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RejectedFitRow { + pub row_id: u64, + pub cluster_id: u64, + pub x: f64, + pub y: f64, + pub sigma_x: f64, + pub sigma_y: f64, + pub fit_residual: f64, + pub n_events: u64, + pub polarity_balance: f64, + pub rejection_reason: RejectionReason, + pub timestamp_us: u64, + pub span_start_us: u64, + pub span_end_us: u64, +} diff --git a/plugins/evesmlm-postproc/README.md b/plugins/evesmlm-postproc/README.md index 932624f..03c5116 100644 --- a/plugins/evesmlm-postproc/README.md +++ b/plugins/evesmlm-postproc/README.md @@ -32,11 +32,20 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global ## Published Data -Publishes filtered and drift-corrected `EveLocalizationResults` on `augur.evesmlm.localization_results`, republishes standard `LocalizationResults` on `augur.localization.results` for downstream compatibility, and serves the compact host-view dataset `augur.evesmlm.current_localizations`. +Publishes filtered and drift-corrected `EveLocalizationResults` on `augur.evesmlm.localization_results`, republishes standard `LocalizationResults` on `augur.localization.results` for downstream compatibility, and serves the shared host-view dataset `augur.evesmlm.current_localizations`. ## Host View -This plugin deliberately reuses the same dataset id and compact panel view id as `EVE Candidate Fitting`. Because post-processing resolves later in the pipeline, it becomes the active provider whenever it is enabled. +This plugin deliberately reuses the same dataset id and view ids as `EVE Candidate Fitting`. + +The shared current-localizations contract includes: + +- stable row ids +- timestamps +- 2D and 3D coordinate metadata +- layer/display metadata + +Because post-processing resolves later in the pipeline, it becomes the active provider whenever it is enabled. ## Dependencies diff --git a/plugins/evesmlm-postproc/src/lib.rs b/plugins/evesmlm-postproc/src/lib.rs index 4d96c65..4413f9d 100644 --- a/plugins/evesmlm-postproc/src/lib.rs +++ b/plugins/evesmlm-postproc/src/lib.rs @@ -10,14 +10,15 @@ pub mod filtering; use std::collections::VecDeque; use augur_plugin_api::{ - export_plugin, AnalysisSeverity, EventStoreHandle, FfiSubpixelMarker, GlobalSettings, - HostContext, HostOutput, HostViewRegistry, Plugin, PluginFrame, PluginInput, SettingItem, - SettingKind, SettingsSchema, SettingsSection, StatusEntry, CTX_GLOBAL_SETTINGS, + export_plugin, AnalysisSeverity, EventStoreHandle, FfiColorRgba, FfiMarkerOverlayItem, + FfiMarkerShape, GlobalSettings, HostContext, HostOutput, HostViewRegistry, Plugin, PluginFrame, + PluginInput, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, + CTX_GLOBAL_SETTINGS, }; pub use augur_plugin_evesmlm_fitting::{ - current_localizations_dataset, current_localizations_registry, to_localization_results, - EveLocalization, EveLocalizationResults, FitMethod, CTX_EVE_LOCALIZATION_RESULTS, - CURRENT_LOCALIZATIONS_DATASET_ID, + current_localizations_dataset, current_localizations_registry_for_results, localization_row_id, + to_localization_results, EveLocalization, EveLocalizationResults, FitMethod, + CTX_EVE_LOCALIZATION_RESULTS, CURRENT_LOCALIZATIONS_DATASET_ID, CURRENT_LOCALIZATIONS_LAYER_ID, }; use augur_plugin_types::CTX_LOCALIZATION_RESULTS; use evaluation::EvaluationState; @@ -63,6 +64,7 @@ pub struct EveSmlmPostProcPlugin { enabled: bool, settings: PostProcSettings, current_results: EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, corrected_history: VecDeque>, evaluation: EvaluationState, last_input_count: usize, @@ -78,6 +80,7 @@ impl Default for EveSmlmPostProcPlugin { enabled: false, settings: PostProcSettings::default(), current_results: EveLocalizationResults::default(), + sensor_dims: None, corrected_history: VecDeque::new(), evaluation: EvaluationState::default(), last_input_count: 0, @@ -91,13 +94,16 @@ impl Default for EveSmlmPostProcPlugin { } impl EveSmlmPostProcPlugin { - fn sync_runtime_settings(&mut self, context: &HostContext<'_>) { + fn sync_runtime_settings(&mut self, context: &HostContext<'_>, frame: &PluginFrame<'_>) { if let Some(settings) = context .get::(CTX_GLOBAL_SETTINGS) .ok() .flatten() { self.settings.nm_per_pixel = settings.nm_per_pixel; + self.sensor_dims = Some((settings.sensor_width, settings.sensor_height)); + } else { + self.sensor_dims = Some((frame.width(), frame.height())); } } @@ -159,15 +165,34 @@ impl EveSmlmPostProcPlugin { self.evaluation.update(&corrected); if self.settings.show_overlay && !corrected.localizations.is_empty() { - let markers: Vec = corrected + let stable_ids: Vec = corrected .localizations .iter() - .map(|localization| FfiSubpixelMarker { + .map(|localization| localization_row_id(localization).to_string()) + .collect(); + let markers: Vec = corrected + .localizations + .iter() + .zip(stable_ids.iter()) + .map(|(localization, stable_id)| FfiMarkerOverlayItem { x: localization.x as f32, y: localization.y as f32, + shape: FfiMarkerShape::Cross, + size: 5.5, + color: FfiColorRgba::from_rgba(OVERLAY_COLOR), + timestamp_us: localization.timestamp_us, + has_timestamp: true, + stable_id: stable_id.as_str().into(), + source_dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + source_row_id: stable_id.as_str().into(), }) .collect(); - output.add_crosshair_markers(&markers, OVERLAY_COLOR, 4); + output.add_marker_overlay( + &markers, + Some(CURRENT_LOCALIZATIONS_DATASET_ID), + Some(CURRENT_LOCALIZATIONS_LAYER_ID), + Some(self.name()), + ); } let mut status = format!( @@ -192,6 +217,7 @@ impl EveSmlmPostProcPlugin { pub fn reset(&mut self) { self.current_results = EveLocalizationResults::default(); + self.sensor_dims = None; self.corrected_history.clear(); self.evaluation.reset(); self.last_input_count = 0; @@ -270,12 +296,12 @@ impl Plugin for EveSmlmPostProcPlugin { fn process_frame( &mut self, - _frame: &PluginFrame<'_>, + frame: &PluginFrame<'_>, output: &mut HostOutput<'_>, context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - self.sync_runtime_settings(context); + self.sync_runtime_settings(context, frame); let input = match context.get::(CTX_EVE_LOCALIZATION_RESULTS) { Ok(value) => value, Err(err) => { @@ -592,7 +618,7 @@ impl Plugin for EveSmlmPostProcPlugin { } fn host_views(&self) -> HostViewRegistry { - current_localizations_registry() + current_localizations_registry_for_results(&self.current_results, self.sensor_dims) } fn host_view_dataset(&self, dataset_id: &str) -> Option> { @@ -618,11 +644,14 @@ mod tests { fn localization(x: f64, y: f64, n_events: usize) -> EveLocalization { EveLocalization { + cluster_id: x.to_bits() ^ y.to_bits(), x, y, sigma_x: 1.2, sigma_y: 1.2, timestamp_us: 0, + span_start_us: 0, + span_end_us: 0, n_events, polarity_balance: 0.0, fit_residual: 0.1, @@ -659,6 +688,22 @@ mod tests { assert!(correction.1.abs() <= 0.1); } + #[test] + fn current_localizations_descriptor_matches_fitting() { + use augur_plugin_evesmlm_fitting::current_localizations_registry_for_results as fitting_registry; + let results = EveLocalizationResults::default(); + let fitting = fitting_registry(&results, None); + let postproc = current_localizations_registry_for_results(&results, None); + let fitting_json = + serde_json::to_value(&fitting).expect("fitting registry should serialize"); + let postproc_json = + serde_json::to_value(&postproc).expect("postproc registry should serialize"); + assert_eq!( + fitting_json, postproc_json, + "postproc must mirror fitting's current_localizations descriptor byte-for-byte", + ); + } + #[test] fn enena_accumulation_collects_expected_nearest_neighbor_distances() { let mut evaluation = EvaluationState::default(); diff --git a/plugins/localization/src/lib.rs b/plugins/localization/src/lib.rs index c542b38..9367630 100644 --- a/plugins/localization/src/lib.rs +++ b/plugins/localization/src/lib.rs @@ -467,7 +467,7 @@ fn build_analysis_image(frame: &PluginFrame<'_>, raw_events: Option<&[FfiCdEvent } let idx = event.y as usize * frame.width() as usize + event.x as usize; let weight = event - .timestamp + .timestamp_us() .saturating_sub(frame.window_start_us()) .max(1) as f64; if event.polarity != 0 { @@ -807,7 +807,7 @@ fn estimate_timestamp_us( continue; } let weight = 1.0 / (1.0 + dist2); - weighted_timestamp += event.timestamp as f64 * weight; + weighted_timestamp += event.timestamp_us() as f64 * weight; weight_sum += weight; } diff --git a/plugins/reconstruction/README.md b/plugins/reconstruction/README.md index d2726b6..b333429 100644 --- a/plugins/reconstruction/README.md +++ b/plugins/reconstruction/README.md @@ -18,12 +18,13 @@ AugurRS now publishes host-owned calibration on `CTX_GLOBAL_SETTINGS` as `Global ## Host Views -The plugin publishes one dataset, `augur.localization.accumulated`, and two host-rendered window views over that dataset: +The plugin publishes one dataset, `augur.localization.accumulated`, and three host-rendered views over that dataset: - `Localization Table` - `Reconstruction` +- `Localization Cloud` -Both views read the same accumulated source of truth. +The dataset now carries stable row ids, timestamps, nanometer-space 2D coordinates, generic 3D scatter metadata, and layer/display metadata. All views read the same accumulated source of truth. ## Compatibility diff --git a/plugins/reconstruction/src/lib.rs b/plugins/reconstruction/src/lib.rs index de404b4..9d03646 100644 --- a/plugins/reconstruction/src/lib.rs +++ b/plugins/reconstruction/src/lib.rs @@ -12,8 +12,10 @@ use std::collections::VecDeque; const DEFAULT_NM_PER_PIXEL: f64 = 65.0; const DEFAULT_MAX_LOCALIZATIONS: usize = 1_000_000; const ACCUMULATED_DATASET_ID: &str = "augur.localization.accumulated"; +const ACCUMULATED_LAYER_ID: &str = "augur.layer.localization.accumulated"; const LOCALIZATION_TABLE_VIEW_ID: &str = "augur.localization.accumulated.table"; const RECONSTRUCTION_VIEW_ID: &str = "augur.localization.accumulated.density"; +const RECONSTRUCTION_3D_VIEW_ID: &str = "augur.localization.accumulated.scatter3d"; #[derive(Debug, Clone)] struct ReconstructionSettings { @@ -149,6 +151,27 @@ impl ReconstructionPlugin { }) } + fn accumulated_coordinate_space_3d(&self) -> Option { + let (sensor_width, sensor_height) = self.sensor_dims?; + let z_min = self.table.front()?.timestamp_us as f64; + let z_max = self + .table + .back()? + .timestamp_us + .max(self.table.front()?.timestamp_us) as f64; + Some(augur_plugin_api::TableCoordinateSpace3d { + x_column: "x_nm".into(), + y_column: "y_nm".into(), + z_column: "timestamp_us".into(), + x_min: 0.0, + x_max: f64::from(sensor_width) * self.settings.nm_per_pixel, + y_min: 0.0, + y_max: f64::from(sensor_height) * self.settings.nm_per_pixel, + z_min, + z_max, + }) + } + fn accumulated_schema(&self) -> augur_plugin_api::TableSchema { augur_plugin_api::TableSchema { columns: vec![ @@ -199,6 +222,71 @@ impl ReconstructionPlugin { }, ], coordinate_space_2d: self.accumulated_coordinate_space(), + coordinate_space_3d: self.accumulated_coordinate_space_3d(), + row_id_column: Some("id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(ACCUMULATED_LAYER_ID.into()), + semantic_label: Some("localizations".into()), + provenance: Some(augur_plugin_api::TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("timestamp_us".into()), + span_end_column: Some("timestamp_us".into()), + anchor_frame_column: Some("frame".into()), + }), + column_display: vec![ + augur_plugin_api::TableColumnDisplayEntry { + column_id: "id".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "x_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 1, + }), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "y_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 1, + }), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "sigma_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 2, + }), + ..Default::default() + }, + }, + augur_plugin_api::TableColumnDisplayEntry { + column_id: "uncertainty_xy_nm".into(), + display: augur_plugin_api::TableColumnDisplayMetadata { + format: Some(augur_plugin_api::TableColumnDisplayFormat::FixedPrecision { + digits: 2, + }), + ..Default::default() + }, + }, + ], } } @@ -269,6 +357,14 @@ impl ReconstructionPlugin { title: "Accumulated localizations".into(), kind: augur_plugin_api::HostDatasetKind::TableV1(self.accumulated_schema()), empty_message: "No accumulated localizations yet.".into(), + display: Some(augur_plugin_api::HostDatasetDisplayMetadata { + layer_title: Some("Accumulated localizations".into()), + default_visibility: Some(true), + default_color: Some([255, 180, 80, 255]), + default_marker_shape: Some(augur_plugin_api::HostMarkerShape::Circle), + default_size: Some(3.5), + }), + relations: Vec::new(), }], views: vec![ augur_plugin_api::HostViewDescriptor { @@ -288,7 +384,19 @@ impl ReconstructionPlugin { y_column: "y_nm".into(), }, }, + augur_plugin_api::HostViewDescriptor { + id: RECONSTRUCTION_3D_VIEW_ID.into(), + title: "Localization Cloud".into(), + dataset_id: ACCUMULATED_DATASET_ID.into(), + placement: augur_plugin_api::HostViewPlacement::Window, + kind: augur_plugin_api::HostViewKind::Scatter3dFromTable { + x_column: "x_nm".into(), + y_column: "y_nm".into(), + z_column: "timestamp_us".into(), + }, + }, ], + actions: Vec::new(), } } } @@ -505,16 +613,23 @@ mod tests { } #[test] - fn host_view_registry_exposes_one_dataset_and_two_window_views() { + fn host_view_registry_exposes_one_dataset_and_investigation_views() { let mut plugin = ReconstructionPlugin::default(); plugin.sensor_dims = Some((1280, 720)); let registry = plugin.host_view_registry(); assert_eq!(registry.datasets.len(), 1); - assert_eq!(registry.views.len(), 2); + assert_eq!(registry.views.len(), 3); assert_eq!(registry.datasets[0].id, ACCUMULATED_DATASET_ID); assert_eq!(registry.views[0].id, LOCALIZATION_TABLE_VIEW_ID); assert_eq!(registry.views[1].id, RECONSTRUCTION_VIEW_ID); + assert_eq!(registry.views[2].id, RECONSTRUCTION_3D_VIEW_ID); + let schema = match ®istry.datasets[0].kind { + augur_plugin_api::HostDatasetKind::TableV1(schema) => schema, + other => panic!("unexpected dataset kind: {other:?}"), + }; + assert_eq!(schema.row_id_column.as_deref(), Some("id")); + assert_eq!(schema.time_column.as_deref(), Some("timestamp_us")); } } diff --git a/scripts/install-built-plugins.sh b/scripts/install-built-plugins.sh index 80a3cbc..d39a8be 100755 --- a/scripts/install-built-plugins.sh +++ b/scripts/install-built-plugins.sh @@ -96,6 +96,21 @@ find_library_path() { return 1 } +rewrite_macos_install_name() { + local installed_library_path="$1" + + if [[ "$(uname -s)" != "Darwin" ]]; then + return 0 + fi + + if ! command -v install_name_tool >/dev/null 2>&1; then + echo "warning: install_name_tool not found; leaving ${installed_library_path} with Cargo's build-path dylib id" >&2 + return 0 + fi + + install_name_tool -id "@loader_path/$(basename "${installed_library_path}")" "${installed_library_path}" +} + library_extension="$(library_extension)" mkdir -p "${dest_dir}" @@ -128,7 +143,9 @@ for plugin_dir in "${repo_root}"/plugins/*; do install_dir="${dest_dir}/${plugin_id}" mkdir -p "${install_dir}" cp "${manifest_path}" "${install_dir}/plugin.toml" - cp "${library_path}" "${install_dir}/$(basename "${library_path}")" + installed_library_path="${install_dir}/$(basename "${library_path}")" + cp "${library_path}" "${installed_library_path}" + rewrite_macos_install_name "${installed_library_path}" echo "Installed ${plugin_id} -> ${install_dir}" installed=$((installed + 1)) done From 10dabfe8033278e7c6f1c9e387da3605a515ac26 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 22:53:02 +0200 Subject: [PATCH 06/30] =?UTF-8?q?fix(plugins):=20=F0=9F=90=9B=20rebuild=20?= =?UTF-8?q?legacy=20plugins=20against=20plugin=20ABI=20v5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the new FfiPreviewFrame.external_triggers field to the two test initializers in focus-metrics and evesmlm-candidates; all plugins now compile and test against the current augur-rs plugin API (ABI v5). --- plugins/evesmlm-candidates/src/lib.rs | 1 + plugins/focus-metrics/src/lib.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index e0bc71e..c173d45 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -2114,6 +2114,7 @@ mod tests { events: augur_plugin_api::FfiSlice::from_slice( &[] as &[augur_plugin_api::FfiCdEvent] ), + external_triggers: augur_plugin_api::FfiSlice::default(), window_start_us: self.window_start_us, window_end_us: self.window_start_us + 1, })); diff --git a/plugins/focus-metrics/src/lib.rs b/plugins/focus-metrics/src/lib.rs index 47c5d23..49a69ab 100644 --- a/plugins/focus-metrics/src/lib.rs +++ b/plugins/focus-metrics/src/lib.rs @@ -656,6 +656,7 @@ mod tests { height: 16, pixels: FfiSlice::from_slice(&pixels), events: FfiSlice::default(), + external_triggers: FfiSlice::default(), window_start_us: 0, window_end_us: 1_000, }; From 0a4be4d8bc14d0e64748e4ca6949e0583a10c9cc Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 13 Jul 2026 22:53:33 +0200 Subject: [PATCH 07/30] =?UTF-8?q?chore(plugins):=20=F0=9F=A7=B9=20apply=20?= =?UTF-8?q?rustfmt=20across=20the=20workspace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/evesmlm-candidates/src/lib.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index c173d45..d51776b 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -17,9 +17,9 @@ use augur_plugin_api::{ HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginCapabilities, PluginFrame, PluginInput, PluginStateKind, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, - TableColumnDisplayEntry, TableColumnDisplayFormat, TableColumnDisplayMetadata, TableColumnValues, - TableColumnWidthPriority, TableCoordinateSpace2d, TableCoordinateSpace3d, TableDatasetV1, - TableRowProvenance, TableSchema, TableValueType, + TableColumnDisplayEntry, TableColumnDisplayFormat, TableColumnDisplayMetadata, + TableColumnValues, TableColumnWidthPriority, TableCoordinateSpace2d, TableCoordinateSpace3d, + TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, }; use serde_json::{json, Value}; @@ -63,8 +63,7 @@ const CANDIDATE_FINDING_PIXELS_DATASET_ID: &str = const CANDIDATE_FINDINGS_LAYER_ID: &str = "augur.layer.evesmlm.candidate_findings"; const CANDIDATE_FINDINGS_COMPACT_VIEW_ID: &str = "augur.evesmlm.candidates.candidate_findings.compact"; -const CANDIDATE_FINDINGS_TABLE_VIEW_ID: &str = - "augur.evesmlm.candidates.candidate_findings.table"; +const CANDIDATE_FINDINGS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.candidate_findings.table"; const CANDIDATE_FINDING_PIXELS_TABLE_VIEW_ID: &str = "augur.evesmlm.candidates.candidate_finding_pixels.table"; @@ -248,11 +247,12 @@ impl EveSmlmCandidatePlugin { } let method = self.settings.finding_method; - self.findings - .extend(clusters.iter().cloned().map(|cluster| CandidateFinding { - cluster, - method, - })); + self.findings.extend( + clusters + .iter() + .cloned() + .map(|cluster| CandidateFinding { cluster, method }), + ); self.findings_generation = self.findings_generation.wrapping_add(1); } From 9c0f349937daf06837d01d8b2b8ae0f355a43f8b Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 14 Jul 2026 13:43:46 +0200 Subject: [PATCH 08/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20align=20mo?= =?UTF-8?q?ck=20and=20host=20plugins=20with=20firmware=200.2.0=20protocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mock controller spoke an invented protocol (ARM/RUN/FAULT_CLEAR verbs, capabilities HELLO field, BAD_* error codes, arbitrary CONFIG fields), so tests validated commands the Teensy never accepts. It now mirrors stage-a-controller/src/main.cpp verbatim: verbs, SAFE_IDLE/CONFIGURED/RUNNING state machine, RANGE/STATE/SYNTAX/PROTOCOL/VERB error details, single-entry idempotent reply cache, and unknown-CONFIG-field rejection (the host's feature-detection contract). The reserved v2 waveform fields are only accepted behind an explicit with_waveform_extension() opt-in, which also synthesizes photodiode blocks through a Pockels-like sin² transfer. Host fixes uncovered by the faithful mock: - surface async control notices (watchdog !FAULT) through poll_events even with no request in flight; monitor and A1 now react instead of showing a stale acquiring state - A1 sweep issues STOP before CONFIG between measurement points (CONFIG is illegal while RUNNING) - record the ACKed CONFIG fields in the A1 run sidecar - reject wire frames with an unknown protocol version like the reference parser does - stream the PDQ file CRC incrementally instead of buffering the whole recording in memory - monitor maps the unknown_config_field rejection of drive fields to a clear 'no waveform backend' message --- plugins/stage-a-a1/src/lib.rs | 40 +- plugins/stage-a-monitor/src/lib.rs | 27 +- stage-a-io/src/client.rs | 70 +++- stage-a-io/src/lib.rs | 1 + stage-a-io/src/mock.rs | 606 +++++++++++++++++++++++++---- stage-a-io/src/pdq.rs | 10 +- stage-a-io/src/wire.rs | 28 ++ 7 files changed, 668 insertions(+), 114 deletions(-) diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs index 379a7f8..808be04 100644 --- a/plugins/stage-a-a1/src/lib.rs +++ b/plugins/stage-a-a1/src/lib.rs @@ -122,6 +122,8 @@ pub struct StageAA1Plugin { reference_counts: Vec, sensor_size: (u16, u16), run_id: String, + /// CONFIG reply fields exactly as the controller ACKed them (sidecar). + last_acked_config: BTreeMap, pdq: Option, current_phase_histogram: Option, used_hardware_fiducial: bool, @@ -159,6 +161,7 @@ impl Default for StageAA1Plugin { reference_counts: Vec::new(), sensor_size: (0, 0), run_id: String::new(), + last_acked_config: BTreeMap::new(), pdq: None, current_phase_histogram: None, used_hardware_fiducial: false, @@ -290,6 +293,7 @@ impl StageAA1Plugin { sidecar.firmware_version = self.firmware.clone(); sidecar.adc_calibration = self.calibration.clone(); sidecar.configured_sample_rate_hz = self.sample_rate_hz as u32; + sidecar.acked_config = self.last_acked_config.clone(); sidecar.trigger_source = if self.used_hardware_fiducial { TriggerSource::DrivePhase0 } else { @@ -321,6 +325,10 @@ impl StageAA1Plugin { fn send_drive(&mut self, frequency_hz: f64, amplitude_dac: u32, purpose: &str) { let freq_mhz = (frequency_hz * 1_000.0).round() as i64; + // The firmware only accepts CONFIG from SAFE_IDLE/CONFIGURED, so + // every new drive point must stop the running acquisition first + // (STOP is idempotent and harmless before the first point). + self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); self.queue_command( purpose, Command::new("CONFIG") @@ -346,19 +354,26 @@ impl StageAA1Plugin { }; let outputs = worker.drain_outputs(); let mut stopped = None; + let mut watchdog_fault: Option = None; for output in outputs { match output { WorkerOutput::Reply { tag, result } => { let purpose = self.in_flight.remove(&tag).unwrap_or_default(); match result { - Ok(fields) => { - if purpose == "hello" { + Ok(fields) => match purpose.as_str() { + "hello" => { self.firmware = fields .get("firmware") .cloned() .unwrap_or_else(|| "unknown".into()); } - } + // CONFIG ACKs (drive points) go into the sidecar + // verbatim, per the control-software spec. + "reference" | "sweep" => { + self.last_acked_config = fields; + } + _ => {} + }, Err(err) => self.last_error = Some(format!("{purpose}: {err}")), } } @@ -372,11 +387,28 @@ impl StageAA1Plugin { } } } - WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { + if name == "FAULT" { + watchdog_fault = Some( + fields + .get("code") + .cloned() + .unwrap_or_else(|| "unknown".into()), + ); + } + } WorkerOutput::Integrity(integrity) => self.integrity = integrity, WorkerOutput::Stopped { reason } => stopped = Some(reason), } } + if let Some(code) = watchdog_fault { + // The controller safed itself mid-run; the current point is + // invalid and the run cannot silently continue. + self.last_error = Some(format!("controller fault: {code} — run aborted")); + if matches!(self.state, RunState::Reference | RunState::Sweeping) { + self.stop_run("watchdog_fault"); + } + } if let Some(reason) = stopped { self.worker = None; self.last_error = Some(format!("device connection ended: {reason}")); diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs index ab159aa..6b3e3e9 100644 --- a/plugins/stage-a-monitor/src/lib.rs +++ b/plugins/stage-a-monitor/src/lib.rs @@ -218,6 +218,15 @@ impl StageAMonitorPlugin { let purpose = self.in_flight.remove(&tag).unwrap_or_default(); match result { Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) if err.contains("unknown_config_field") => { + // Feature detection: firmware 0.2.0 has no + // waveform backend and rejects the reserved v2 + // drive fields. + self.last_error = Some(format!( + "{purpose}: firmware has no waveform backend (v1) — drive \ + control needs the mock or the future v2 firmware" + )); + } Err(err) => { self.last_error = Some(format!("{purpose}: {err}")); } @@ -233,7 +242,23 @@ impl StageAMonitorPlugin { FrameType::Summary | FrameType::Marker | FrameType::Control => {} FrameType::Unknown(_) => {} }, - WorkerOutput::Event(DeviceEvent::Async { .. }) => {} + WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { + if name == "FAULT" { + // Firmware watchdog dropped the controller to + // SAFE_IDLE — reflect it instead of showing a stale + // "acquiring" state. + if self.connection == ConnectionState::Acquiring { + self.connection = ConnectionState::Connected; + } + self.last_error = Some(format!( + "controller fault: {} — dropped to SAFE_IDLE", + fields.get("code").map(String::as_str).unwrap_or("unknown") + )); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + } WorkerOutput::Integrity(integrity) => { self.integrity = integrity; } diff --git a/stage-a-io/src/client.rs b/stage-a-io/src/client.rs index c664be0..0b018bb 100644 --- a/stage-a-io/src/client.rs +++ b/stage-a-io/src/client.rs @@ -189,9 +189,19 @@ impl StageAClient { match frame.header.frame_type { FrameType::Control => { - // Control payloads are handled by take_reply / async queue; - // keep the raw frame so replies can be matched later. - self.pending_events.push(DeviceEvent::Data(frame)); + // Classify control payloads immediately so async notices + // (e.g. the watchdog `!FAULT`) surface through poll_events + // even when no request is in flight. Replies stay queued as + // raw frames for take_reply to match by sequence. + match frame.control_text().map(ControlMessage::parse) { + Some(Ok(ControlMessage::Async { name, fields })) => { + self.pending_events + .push(DeviceEvent::Async { name, fields }); + } + Some(Ok(_)) => self.pending_events.push(DeviceEvent::Data(frame)), + // Non-UTF8 or malformed control payload: corruption. + _ => self.integrity.skipped_bytes += frame.payload.len() as u64, + } } _ => self.pending_events.push(DeviceEvent::Data(frame)), } @@ -230,16 +240,10 @@ impl StageAClient { }) if reply_seq == sequence => { result = Some(Err(ClientError::Device { code, detail })); } - Ok(ControlMessage::Async { name, fields }) => { - remaining.push(DeviceEvent::Async { name, fields }); - } - // Stale replies to earlier (retried) sequences are dropped; - // malformed control payloads count as corruption. - Ok(_) => {} - Err(_) => { - self.integrity.crc_failures += 0; // parse failure, not CRC - self.integrity.skipped_bytes += frame.payload.len() as u64; - } + // Stale replies to earlier (retried) sequences are dropped. + // Async / malformed payloads never reach here — accept_frame + // classifies them before queueing. + _ => {} } } self.pending_events = remaining; @@ -285,14 +289,17 @@ mod tests { // The controller swallows the first reply; the client must resend the // identical sequence and accept the cached second reply. The mock // panics if a retried sequence re-executes the operation. - let handle = std::thread::spawn(move || controller.serve_n_commands(2)); + let handle = std::thread::spawn(move || { + controller.serve_n_commands(2); + controller + }); let reply = client .request(&Command::new("STATUS")) .expect("retried STATUS succeeds"); - handle.join().expect("mock thread joins"); + let controller = handle.join().expect("mock thread joins"); assert_eq!(reply.get("state").map(String::as_str), Some("SAFE_IDLE")); - assert_eq!(reply.get("executions").map(String::as_str), Some("1")); + assert_eq!(controller.executions(), 1); } #[test] @@ -304,16 +311,43 @@ mod tests { let handle = std::thread::spawn(move || controller.serve_n_commands(1)); let err = client - .request(&Command::new("CONFIG").field("mode", "A9")) + .request( + &Command::new("CONFIG") + .field("mode", "A9") + .field("rate_hz", 20_000), + ) .expect_err("invalid mode is rejected"); handle.join().expect("mock thread joins"); match err { - ClientError::Device { code, .. } => assert_eq!(code, "BAD_MODE"), + ClientError::Device { code, detail } => { + assert_eq!(code, "RANGE"); + assert_eq!(detail, "invalid_mode"); + } other => panic!("expected device error, got {other:?}"), } } + #[test] + fn watchdog_fault_surfaces_as_async_event_without_a_request_in_flight() { + let link = MockLink::new(); + let mut controller = MockController::new(link.device_end()); + let mut client = + StageAClient::new(link.host_end()).with_reply_timeout(Duration::from_millis(100)); + + controller.emit_watchdog_fault(); + let events = client.poll_events().expect("poll"); + match events.as_slice() { + [DeviceEvent::Async { name, fields }] => { + assert_eq!(name, "FAULT"); + assert_eq!(fields.get("code").map(String::as_str), Some("WATCHDOG")); + assert_eq!(fields.get("state").map(String::as_str), Some("SAFE_IDLE")); + } + other => panic!("expected one async FAULT event, got {other:?}"), + } + assert!(client.integrity().is_clean()); + } + #[test] fn overrun_frames_invalidate_integrity() { let link = MockLink::new(); diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index 67c517f..2a21ff7 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -31,6 +31,7 @@ pub mod wire; pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; pub use estimator::{estimate_contrast, AdcCalibration, ContrastEstimate, EstimateError}; +pub use mock::{MockController, MockState, MockWave}; pub use pdq::{PdqSummary, PdqWriter}; pub use protocol::{Command, ControlMessage, ProtocolError}; pub use sidecar::{DetectorLoad, IntegrityRecord, RunSidecar, TriggerSource}; diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs index fd7dad0..803d919 100644 --- a/stage-a-io/src/mock.rs +++ b/stage-a-io/src/mock.rs @@ -1,23 +1,34 @@ //! Mock Stage-A controller for tests and hardware-free plugin development. //! -//! Implements the v1 command surface (`HELLO`, `STATUS`, `CONFIG`, `ARM`, -//! `RUN`, `START`, `STOP`, `PING`, `FAULT_CLEAR`) with the same idempotency -//! contract as the firmware: replies to recent sequences are cached and -//! resent without re-executing the operation. It can also synthesize -//! photodiode sample/summary frames (sinusoidal drive) so the estimator and -//! plugins can be exercised end to end without a Teensy. - -use std::collections::BTreeMap; +//! Mirrors firmware 0.2.0 (`stage-a-controller/src/main.cpp`) faithfully: +//! the same verbs (`HELLO`, `STATUS`, `CONFIG`, `START`, `STOP`, `PING`), +//! the same state machine (`SAFE_IDLE` → `CONFIGURED` → `RUNNING`), the +//! same error codes/details (`PROTOCOL`, `RANGE`, `STATE`, `SYNTAX`, +//! `VERB`), the same single-entry idempotent reply cache, and rejection of +//! unknown `CONFIG` fields — which is the host's feature-detection +//! mechanism, so it must never be papered over here. +//! +//! [`MockController::with_waveform_extension`] additionally models the +//! *proposed* v2 waveform firmware (`stage-a-controller/docs/features/` +//! `waveform-drive.md`): `wave`/`freq_mhz`/`center_dac`/`amplitude_dac` +//! CONFIG fields, a `capabilities` HELLO entry, and synthetic photodiode +//! blocks derived from the configured drive through a Pockels-like sin² +//! transfer — commanded DAC amplitude maps *non-linearly* to optical +//! contrast, exactly why `a` must be measured, never assumed. use crate::protocol::ControlMessage; use crate::transport::Transport; use crate::wire::{Frame, FrameHeader, FrameType, SummaryPayload, PROTOCOL_VERSION}; +pub const MOCK_MAX_RATE_HZ: u32 = 100_000; +pub const MOCK_MAX_BLOCK_SAMPLES: u32 = 256; +/// Proposed v2 waveform ceiling (matches the drive UI bound: 200 kHz). +pub const MOCK_MAX_FREQ_MHZ: u32 = 200_000_000; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MockState { SafeIdle, Configured, - Armed, Running, } @@ -26,29 +37,87 @@ impl MockState { match self { Self::SafeIdle => "SAFE_IDLE", Self::Configured => "CONFIGURED", - Self::Armed => "ARMED", Self::Running => "RUNNING", } } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MockWave { + Sine, + Square, + Saw, +} + +impl MockWave { + /// Normalised waveform value in [-1, 1] at cycle phase `t` in [0, 1). + fn value(self, t: f64) -> f64 { + match self { + Self::Sine => (2.0 * std::f64::consts::PI * t).sin(), + Self::Square => { + if t < 0.5 { + 1.0 + } else { + -1.0 + } + } + Self::Saw => 2.0 * t - 1.0, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +struct MockConfig { + mode: String, + rate_hz: u32, + block_samples: u32, + raw: bool, + summary: bool, + // v2 waveform extension (None until configured). + wave: Option, + freq_mhz: u32, + center_dac: u32, + amplitude_dac: u32, +} + +impl Default for MockConfig { + fn default() -> Self { + Self { + mode: "A1".into(), + rate_hz: 20_000, + block_samples: 256, + raw: true, + summary: true, + wave: None, + freq_mhz: 0, + center_dac: 2_048, + amplitude_dac: 0, + } + } +} + pub struct MockController { transport: T, state: MockState, - config: BTreeMap, - config_revision: u32, - reply_cache: Vec<(u32, String)>, + config: MockConfig, + /// v2 waveform CONFIG fields accepted (proposed firmware) instead of + /// rejected as `unknown_config_field` (firmware 0.2.0). + waveform_extension: bool, + /// Firmware caches exactly one reply (`cached_request_sequence`). + cached_reply: Option<(u32, String)>, executed_sequences: Vec, - /// Commands executed (used to assert idempotency in tests). executions: u32, drop_next_reply: bool, out_sequence: u32, line_buffer: Vec, sample_index: u64, - /// Synthetic optical waveform: codes = center + amplitude*sin(phase). + /// Synthetic optics for [`MockController::emit_configured_block`]: + /// photodiode code = dark + span * sin²(π/2 · drive/4095). + pub synth_dark_code: f64, + pub synth_span_codes: f64, + // Legacy direct-sine synthesis (emit_sine_block). pub synth_center: f64, pub synth_amplitude: f64, - pub synth_dark_code: f64, } impl MockController { @@ -56,21 +125,28 @@ impl MockController { Self { transport, state: MockState::SafeIdle, - config: BTreeMap::new(), - config_revision: 0, - reply_cache: Vec::new(), + config: MockConfig::default(), + waveform_extension: false, + cached_reply: None, executed_sequences: Vec::new(), executions: 0, drop_next_reply: false, out_sequence: 0, line_buffer: Vec::new(), sample_index: 0, + synth_dark_code: 40.0, + synth_span_codes: 3_800.0, synth_center: 2_048.0, synth_amplitude: 900.0, - synth_dark_code: 40.0, } } + /// Enables the proposed v2 waveform command surface. + pub fn with_waveform_extension(mut self) -> Self { + self.waveform_extension = true; + self + } + /// Swallow the next reply (simulates a lost USB packet) — the client /// must retry with the identical sequence. pub fn drop_first_reply(&mut self) { @@ -81,6 +157,41 @@ impl MockController { self.state } + /// Commands actually executed (idempotent retries excluded). + pub fn executions(&self) -> u32 { + self.executions + } + + /// Handles all complete command lines already received, without + /// blocking — for long-lived in-process mock threads (e.g. a plugin's + /// hardware-free `mock` port). + pub fn poll_commands(&mut self) { + let mut buf = [0_u8; 1024]; + loop { + let read = self.transport.read(&mut buf).unwrap_or(0); + if read == 0 { + break; + } + self.line_buffer.extend_from_slice(&buf[..read]); + } + while let Some(pos) = self.line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = self.line_buffer.drain(..=pos).collect(); + if let Ok(text) = std::str::from_utf8(&line) { + let text = text.trim_end().to_owned(); + self.handle_line(&text); + } + } + } + + /// Wall-clock duration one configured sample block spans — the cadence + /// at which a live mock should call [`Self::emit_configured_block`]. + pub fn block_period(&self) -> std::time::Duration { + let rate = self.config.rate_hz.max(1); + std::time::Duration::from_micros( + u64::from(self.config.block_samples) * 1_000_000 / u64::from(rate), + ) + } + /// Serves exactly `n` command lines (counting retries), then returns. pub fn serve_n_commands(&mut self, n: usize) { let mut served = 0; @@ -108,21 +219,21 @@ impl MockController { fn handle_line(&mut self, line: &str) { let Some(rest) = line.strip_prefix('@') else { + self.send_control("-0 ERR code=SYNTAX detail=expected_sequence_and_verb"); return; }; let mut parts = rest.split_ascii_whitespace(); let Some(sequence) = parts.next().and_then(|s| s.parse::().ok()) else { + self.send_control("-0 ERR code=SYNTAX detail=invalid_sequence"); return; }; // Idempotent retry: replay the cached reply without re-executing. - if let Some((_, cached)) = self - .reply_cache - .iter() - .find(|(cached_seq, _)| *cached_seq == sequence) - { - let payload = cached.clone(); - self.send_control(&payload); - return; + if let Some((cached_seq, cached)) = &self.cached_reply { + if *cached_seq == sequence { + let payload = cached.clone(); + self.send_control(&payload); + return; + } } assert!( !self.executed_sequences.contains(&sequence), @@ -130,7 +241,8 @@ impl MockController { ); let verb = parts.next().unwrap_or(""); - let fields: BTreeMap = parts + // Preserve wire order: firmware validates fields as encountered. + let fields: Vec<(String, String)> = parts .filter_map(|part| { let (key, value) = part.split_once('=')?; Some((key.to_owned(), value.to_owned())) @@ -140,10 +252,7 @@ impl MockController { self.executions += 1; self.executed_sequences.push(sequence); let reply = self.execute(verb, &fields, sequence); - self.reply_cache.push((sequence, reply.clone())); - if self.reply_cache.len() > 8 { - self.reply_cache.remove(0); - } + self.cached_reply = Some((sequence, reply.clone())); if self.drop_next_reply { self.drop_next_reply = false; return; @@ -151,50 +260,147 @@ impl MockController { self.send_control(&reply); } - fn execute(&mut self, verb: &str, fields: &BTreeMap, sequence: u32) -> String { + fn execute(&mut self, verb: &str, fields: &[(String, String)], sequence: u32) -> String { + let field = |key: &str| { + fields + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + }; match verb { - "HELLO" => format!( - "+{sequence} OK protocol=1 firmware=0.1.0-mock board=mock dac_bits=12 \ - capabilities=A1,A2,A3" - ), + "HELLO" => { + if field("protocol") != Some("1") { + return format!("-{sequence} ERR code=PROTOCOL detail=requires_v1"); + } + let capabilities = if self.waveform_extension { + " capabilities=A1,A2,A3,WAVE" + } else { + "" + }; + format!( + "+{sequence} OK protocol=1 firmware=0.2.0-mock board=MOCK adc_bits=12 \ + max_rate_hz={MOCK_MAX_RATE_HZ} dac=AD5628 dac_bus=SPI1 dac_cs=29 \ + dac_channel=1.4 dac_address=3{capabilities}" + ) + } "STATUS" => format!( - "+{sequence} OK state={} rev={} executions={}", + "+{sequence} OK state={} mode={} rate_hz={} block_samples={} raw={} summary={} \ + sample_index={} dropped=0 marker_drops=0 dac=1.4/3 code=0", self.state.name(), - self.config_revision, - self.executions + self.config.mode, + self.config.rate_hz, + self.config.block_samples, + u8::from(self.config.raw), + u8::from(self.config.summary), + self.sample_index, ), - "PING" => format!("+{sequence} OK state={}", self.state.name()), - "CONFIG" => { - let mode = fields.get("mode").map(String::as_str).unwrap_or(""); - if !matches!(mode, "A1" | "A2" | "A3") { - return format!("-{sequence} ERR code=BAD_MODE detail=mode"); - } - self.config = fields.clone(); - self.config_revision += 1; - self.state = MockState::Configured; - format!("+{sequence} OK rev={}", self.config_revision) - } - "ARM" => { + "CONFIG" => self.execute_config(fields, sequence), + "START" => { if self.state != MockState::Configured { - return format!("-{sequence} ERR code=BAD_STATE detail=arm_requires_config"); - } - self.state = MockState::Armed; - format!("+{sequence} OK state=ARMED rev={}", self.config_revision) - } - "RUN" | "START" => { - if !matches!(self.state, MockState::Armed | MockState::Configured) { - return format!("-{sequence} ERR code=BAD_STATE detail=run_requires_arm"); + return format!("-{sequence} ERR code=STATE detail=configure_before_start"); } self.state = MockState::Running; format!("+{sequence} OK state=RUNNING") } + // Firmware ignores extra STOP tokens (e.g. reason=…). "STOP" => { self.state = MockState::SafeIdle; format!("+{sequence} OK state=SAFE_IDLE") } - "FAULT_CLEAR" => format!("+{sequence} OK state={}", self.state.name()), - _ => format!("-{sequence} ERR code=BAD_VERB detail={verb}"), + "PING" => format!("+{sequence} OK watchdog=refreshed"), + _ => format!("-{sequence} ERR code=VERB detail=unsupported_command"), + } + } + + fn execute_config(&mut self, fields: &[(String, String)], sequence: u32) -> String { + if self.state == MockState::Running { + return format!("-{sequence} ERR code=STATE detail=stop_before_config"); + } + let err = |code: &str, detail: &str| format!("-{sequence} ERR code={code} detail={detail}"); + let mut next = self.config.clone(); + let mut saw_mode = false; + let mut saw_rate = false; + for (key, value) in fields { + match key.as_str() { + "mode" => { + saw_mode = true; + if !matches!(value.as_str(), "A1" | "A2" | "A3") { + return err("RANGE", "invalid_mode"); + } + next.mode = value.clone(); + } + "rate_hz" => { + saw_rate = true; + match value.parse::() { + Ok(rate) if (100..=MOCK_MAX_RATE_HZ).contains(&rate) => { + next.rate_hz = rate; + } + _ => return err("RANGE", "invalid_rate_hz"), + } + } + "block_samples" => match value.parse::() { + Ok(block) if (1..=MOCK_MAX_BLOCK_SAMPLES).contains(&block) => { + next.block_samples = block; + } + _ => return err("RANGE", "invalid_block_samples"), + }, + "raw" => match value.as_str() { + "0" => next.raw = false, + "1" => next.raw = true, + _ => return err("RANGE", "invalid_raw_flag"), + }, + "summary" => match value.as_str() { + "0" => next.summary = false, + "1" => next.summary = true, + _ => return err("RANGE", "invalid_summary_flag"), + }, + "wave" if self.waveform_extension => { + next.wave = Some(match value.as_str() { + "SINE" => MockWave::Sine, + "SQUARE" => MockWave::Square, + "SAW" => MockWave::Saw, + _ => return err("RANGE", "invalid_wave"), + }); + } + "freq_mhz" if self.waveform_extension => match value.parse::() { + Ok(freq) if (1..=MOCK_MAX_FREQ_MHZ).contains(&freq) => { + next.freq_mhz = freq; + } + _ => return err("RANGE", "invalid_freq_mhz"), + }, + "center_dac" if self.waveform_extension => match value.parse::() { + Ok(center) if center <= 4_095 => next.center_dac = center, + _ => return err("RANGE", "invalid_center_dac"), + }, + "amplitude_dac" if self.waveform_extension => match value.parse::() { + Ok(amplitude) if amplitude <= 2_047 => next.amplitude_dac = amplitude, + _ => return err("RANGE", "invalid_amplitude_dac"), + }, + // Firmware 0.2.0 rejects unknown fields — the host relies + // on this for feature detection. Never accept silently. + _ => return err("SYNTAX", "unknown_config_field"), + } + } + if !saw_mode || !saw_rate || (!next.raw && !next.summary) { + return err("SYNTAX", "mode_rate_and_output_required"); } + if next.wave.is_some() + && (next.center_dac + next.amplitude_dac > 4_095 + || next.center_dac < next.amplitude_dac) + { + return err("RANGE", "amplitude_exceeds_range"); + } + self.config = next; + self.state = MockState::Configured; + format!( + "+{sequence} OK state=CONFIGURED mode={} rate_hz={} block_samples={} raw={} \ + summary={} backend=mock", + self.config.mode, + self.config.rate_hz, + self.config.block_samples, + u8::from(self.config.raw), + u8::from(self.config.summary), + ) } fn send_control(&mut self, payload: &str) { @@ -203,6 +409,13 @@ impl MockController { let _ = self.transport.write_all(&bytes); } + /// Emits the watchdog fault notice and drops to `SAFE_IDLE`, exactly as + /// the firmware does after 1.5 s without host contact. + pub fn emit_watchdog_fault(&mut self) { + self.state = MockState::SafeIdle; + self.send_control("!FAULT code=WATCHDOG state=SAFE_IDLE"); + } + fn build_frame( &mut self, frame_type: FrameType, @@ -227,38 +440,83 @@ impl MockController { ) } - /// Emits one synthetic sinusoidal sample block (`SamplesU16`). - pub fn emit_sine_block(&mut self, samples: usize, rate_hz: u32, freq_hz: f64) { - let mut payload = Vec::with_capacity(samples * 2); + fn emit_codes_block(&mut self, codes: &[u16], rate_hz: u32, raw: bool, summary: bool) { let mut min_code = u16::MAX; let mut max_code = 0_u16; let mut sum = 0_u64; - for i in 0..samples { - let t = (self.sample_index + i as u64) as f64 / f64::from(rate_hz); - let value = self.synth_center - + self.synth_amplitude * (2.0 * std::f64::consts::PI * freq_hz * t).sin(); - let code = value.round().clamp(0.0, 4_095.0) as u16; + let mut payload = Vec::with_capacity(codes.len() * 2); + for &code in codes { min_code = min_code.min(code); max_code = max_code.max(code); sum += u64::from(code); payload.extend_from_slice(&code.to_le_bytes()); } - let frame = self.build_frame(FrameType::SamplesU16, payload, rate_hz, 0); - let bytes = frame.to_bytes(); - let _ = self.transport.write_all(&bytes); + if raw { + let frame = self.build_frame(FrameType::SamplesU16, payload, rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } + if summary { + let summary_payload = SummaryPayload { + min_code, + max_code, + sample_count: codes.len() as u32, + sum_codes: sum, + first_tick_us: 0, + last_tick_us: ((codes.len() as f64 / f64::from(rate_hz)) * 1e6) as u32, + }; + let frame = self.build_frame(FrameType::Summary, summary_payload.encode(), rate_hz, 0); + let bytes = frame.to_bytes(); + let _ = self.transport.write_all(&bytes); + } + self.sample_index += codes.len() as u64; + } - let summary = SummaryPayload { - min_code, - max_code, - sample_count: samples as u32, - sum_codes: sum, - first_tick_us: 0, - last_tick_us: ((samples as f64 / f64::from(rate_hz)) * 1e6) as u32, - }; - let frame = self.build_frame(FrameType::Summary, summary.encode(), rate_hz, 0); - let bytes = frame.to_bytes(); - let _ = self.transport.write_all(&bytes); - self.sample_index += samples as u64; + /// Emits one photodiode block synthesized from the *configured* v2 + /// drive: DAC waveform → Pockels-like sin² intensity transfer → ADC + /// codes. Without a configured `wave` (or with `amplitude_dac = 0`) the + /// output is the flat unmodulated level at `center_dac`. + pub fn emit_configured_block(&mut self) { + if self.state != MockState::Running { + return; + } + let config = self.config.clone(); + let rate = f64::from(config.rate_hz); + let freq_hz = f64::from(config.freq_mhz) / 1_000.0; + let codes: Vec = (0..config.block_samples as u64) + .map(|i| { + let t = (self.sample_index + i) as f64 / rate; + let shape = match (config.wave, config.amplitude_dac) { + (Some(wave), amplitude) if amplitude > 0 && freq_hz > 0.0 => { + wave.value((t * freq_hz).fract()) + } + _ => 0.0, + }; + let drive = f64::from(config.center_dac) + f64::from(config.amplitude_dac) * shape; + let transmission = (std::f64::consts::FRAC_PI_2 * drive / 4_095.0) + .sin() + .powi(2); + (self.synth_dark_code + self.synth_span_codes * transmission) + .round() + .clamp(0.0, 4_095.0) as u16 + }) + .collect(); + self.emit_codes_block(&codes, config.rate_hz, config.raw, config.summary); + } + + /// Emits one synthetic sinusoidal sample block (`SamplesU16` + + /// `Summary`), bypassing the drive model — codes = center + A·sin. + pub fn emit_sine_block(&mut self, samples: usize, rate_hz: u32, freq_hz: f64) { + let codes: Vec = (0..samples as u64) + .map(|i| { + let t = (self.sample_index + i) as f64 / f64::from(rate_hz); + (self.synth_center + + self.synth_amplitude * (2.0 * std::f64::consts::PI * freq_hz * t).sin()) + .round() + .clamp(0.0, 4_095.0) as u16 + }) + .collect(); + self.emit_codes_block(&codes, rate_hz, true, true); } /// Emits a summary frame carrying a nonzero overrun counter. @@ -281,3 +539,179 @@ impl MockController { pub fn parse_control(text: &str) -> Option { ControlMessage::parse(text).ok() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::transport::MockLink; + + fn request(controller: &mut MockController, line: &str) { + let mut bytes = line.as_bytes().to_vec(); + bytes.push(b'\n'); + // Feed the line directly through the device-side buffer path. + controller.line_buffer.extend_from_slice(&bytes); + while let Some(pos) = controller.line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = controller.line_buffer.drain(..=pos).collect(); + let text = std::str::from_utf8(&line).unwrap().trim_end().to_owned(); + controller.handle_line(&text); + } + } + + fn last_control_text(host: &mut crate::transport::MockTransport) -> String { + let mut parser = crate::wire::FrameParser::default(); + let mut buf = [0_u8; 4096]; + let mut last = None; + loop { + let n = crate::transport::Transport::read(host, &mut buf).unwrap(); + if n == 0 { + break; + } + parser.extend(&buf[..n]); + } + while let Some(event) = parser.next_event() { + if let crate::wire::ParseEvent::Frame(frame) = event { + if let Some(text) = frame.control_text() { + last = Some(text.to_owned()); + } + } + } + last.expect("a control frame was emitted") + } + + #[test] + fn matches_firmware_state_machine_and_error_details() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + // START before CONFIG → STATE error, firmware detail string. + request(&mut controller, "@1 START"); + assert!(last_control_text(&mut host).contains("code=STATE detail=configure_before_start")); + + // Valid CONFIG, then START, then CONFIG while running is rejected. + request(&mut controller, "@2 CONFIG mode=A1 rate_hz=20000"); + assert!(last_control_text(&mut host).starts_with("+2 OK state=CONFIGURED")); + request(&mut controller, "@3 START"); + assert_eq!(controller.state(), MockState::Running); + request(&mut controller, "@4 CONFIG mode=A1 rate_hz=20000"); + assert!(last_control_text(&mut host).contains("code=STATE detail=stop_before_config")); + + // STOP always succeeds and ignores extra fields. + request(&mut controller, "@5 STOP reason=test"); + assert_eq!(controller.state(), MockState::SafeIdle); + } + + #[test] + fn firmware_v1_rejects_waveform_fields_as_unknown() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + request( + &mut controller, + "@1 CONFIG mode=A1 wave=SINE freq_mhz=1000000 rate_hz=20000", + ); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=unknown_config_field")); + } + + #[test] + fn hello_requires_protocol_v1_and_advertises_capabilities_only_with_extension() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + request(&mut controller, "@1 HELLO"); + assert!(last_control_text(&mut host).contains("code=PROTOCOL detail=requires_v1")); + request(&mut controller, "@2 HELLO protocol=1"); + assert!(!last_control_text(&mut host).contains("capabilities")); + + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + request(&mut controller, "@1 HELLO protocol=1"); + assert!(last_control_text(&mut host).contains("capabilities=A1,A2,A3,WAVE")); + } + + #[test] + fn waveform_extension_validates_drive_bounds() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + + request( + &mut controller, + "@1 CONFIG mode=A1 rate_hz=20000 wave=TRIANGLE freq_mhz=1000000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=invalid_wave")); + + request( + &mut controller, + "@2 CONFIG mode=A1 rate_hz=20000 wave=SINE freq_mhz=1000000 center_dac=3000 \ + amplitude_dac=2000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=amplitude_exceeds_range")); + + request( + &mut controller, + "@3 CONFIG mode=A1 rate_hz=20000 wave=SAW freq_mhz=1000000 center_dac=2048 \ + amplitude_dac=512", + ); + assert!(last_control_text(&mut host).starts_with("+3 OK state=CONFIGURED")); + } + + #[test] + fn configured_drive_synthesizes_nonlinear_pockels_response() { + let contrast_for_amplitude = |amplitude: u32| -> f64 { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + request( + &mut controller, + &format!( + "@1 CONFIG mode=A1 rate_hz=20000 wave=SINE freq_mhz=100000 center_dac=2048 \ + amplitude_dac={amplitude}" + ), + ); + request(&mut controller, "@2 START"); + let _ = last_control_text(&mut host); + for _ in 0..8 { + controller.emit_configured_block(); + } + + let mut parser = crate::wire::FrameParser::default(); + let mut buf = [0_u8; 65_536]; + loop { + let n = crate::transport::Transport::read(&mut host, &mut buf).unwrap(); + if n == 0 { + break; + } + parser.extend(&buf[..n]); + } + let mut codes = Vec::new(); + while let Some(event) = parser.next_event() { + if let crate::wire::ParseEvent::Frame(frame) = event { + if let Some(samples) = frame.samples() { + codes.extend(samples); + } + } + } + let estimate = crate::estimator::estimate_contrast( + &codes, + &crate::estimator::AdcCalibration { + dark_volts: 40.0 * 3.3 / 4_095.0, + ..Default::default() + }, + ) + .expect("clean synthetic window"); + estimate.a + }; + + let a_small = contrast_for_amplitude(512); + let a_double = contrast_for_amplitude(1_024); + assert!(a_small > 0.0 && a_double > a_small); + // sin² transfer: doubling the DAC amplitude must NOT double a. + assert!( + (a_double / a_small - 2.0).abs() > 0.05, + "a_small={a_small} a_double={a_double} — response looks linear" + ); + } +} diff --git a/stage-a-io/src/pdq.rs b/stage-a-io/src/pdq.rs index 0e2119f..1eb4d1c 100644 --- a/stage-a-io/src/pdq.rs +++ b/stage-a-io/src/pdq.rs @@ -11,14 +11,14 @@ use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use crate::client::StreamIntegrity; -use crate::wire::{crc32, Frame}; +use crate::wire::{Crc32, Frame}; pub struct PdqWriter { path: PathBuf, file: BufWriter, frames_written: u64, bytes_written: u64, - running_crc_bytes: Vec, + running_crc: Crc32, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -43,7 +43,7 @@ impl PdqWriter { path, frames_written: 0, bytes_written: 0, - running_crc_bytes: Vec::new(), + running_crc: Crc32::default(), }) } @@ -52,7 +52,7 @@ impl PdqWriter { self.file.write_all(&bytes)?; self.frames_written += 1; self.bytes_written += bytes.len() as u64; - self.running_crc_bytes.extend_from_slice(&bytes); + self.running_crc.update(&bytes); Ok(()) } @@ -60,7 +60,7 @@ impl PdqWriter { pub fn finish(mut self, integrity: StreamIntegrity) -> std::io::Result { self.file.flush()?; Ok(PdqSummary { - file_crc32: crc32(&self.running_crc_bytes), + file_crc32: self.running_crc.finalize(), path: self.path, frames_written: self.frames_written, bytes_written: self.bytes_written, diff --git a/stage-a-io/src/wire.rs b/stage-a-io/src/wire.rs index 33e8caa..c42a249 100644 --- a/stage-a-io/src/wire.rs +++ b/stage-a-io/src/wire.rs @@ -65,6 +65,11 @@ impl FrameHeader { if magic != MAGIC { return None; } + // Unknown protocol versions are corruption, not future frames: the + // reference host parser resynchronises past them byte by byte. + if bytes[4] != PROTOCOL_VERSION { + return None; + } Some(Self { version: bytes[4], frame_type: FrameType::from_raw(bytes[5]), @@ -190,6 +195,29 @@ pub fn crc32(data: &[u8]) -> u32 { crc32_update(0xFFFF_FFFF, data) ^ 0xFFFF_FFFF } +/// Streaming CRC32 with the same parameters as [`crc32`], for hashing data +/// that is not held in memory at once (e.g. the PDQ file writer). +#[derive(Debug, Clone, Copy)] +pub struct Crc32 { + state: u32, +} + +impl Default for Crc32 { + fn default() -> Self { + Self { state: 0xFFFF_FFFF } + } +} + +impl Crc32 { + pub fn update(&mut self, data: &[u8]) { + self.state = crc32_update(self.state, data); + } + + pub fn finalize(self) -> u32 { + self.state ^ 0xFFFF_FFFF + } +} + fn crc32_update(mut crc: u32, data: &[u8]) -> u32 { for &byte in data { crc ^= u32::from(byte); From 58296eb864e8a70ab93d3c798cc89fcf9579902e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 14 Jul 2026 13:43:56 +0200 Subject: [PATCH 09/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20functi?= =?UTF-8?q?on-generator=20familiarisation=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New plugins/stage-a-funcgen crate: manual Pockels-cell drive control (sine/square/sawtooth, frequency, center/amplitude DAC codes) with the resulting optical amplitude always measured from the photodiode as a = ln(Vmax/Vmin) — the commanded amplitude is a phase-modulation depth and maps non-linearly to light, so it is never reported as an optical level. The default 'mock' port runs the waveform-extended mock controller on an in-process thread and streams a synthetic sin²-transfer photodiode response, so the full control loop works with zero hardware. Against real firmware 0.2.0 the reserved drive fields are feature-detected via the unknown_config_field rejection and reported as 'no waveform backend'; actual output stays blocked on the hardware freeze per stage-a-controller/docs/features/waveform-drive.md. Same fail-closed safety model as stage-a-monitor: LiveCapture + effects gating, drive parameters as settings but application as an explicit action, local DAC-range validation before any command, watchdog fault surfacing. --- Cargo.toml | 1 + docs/features/README.md | 1 + docs/features/stage-a-funcgen.md | 57 ++ docs/features/stage-a.md | 9 +- plugins/stage-a-funcgen/Cargo.toml | 15 + plugins/stage-a-funcgen/README.md | 47 ++ plugins/stage-a-funcgen/plugin.toml | 7 + plugins/stage-a-funcgen/src/lib.rs | 1011 +++++++++++++++++++++++++++ 8 files changed, 1146 insertions(+), 2 deletions(-) create mode 100644 docs/features/stage-a-funcgen.md create mode 100644 plugins/stage-a-funcgen/Cargo.toml create mode 100644 plugins/stage-a-funcgen/README.md create mode 100644 plugins/stage-a-funcgen/plugin.toml create mode 100644 plugins/stage-a-funcgen/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index e2b3d5a..3c96d7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "stage-a-io", "plugins/stage-a-monitor", "plugins/stage-a-a1", + "plugins/stage-a-funcgen", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", diff --git a/docs/features/README.md b/docs/features/README.md index 02e8e28..39e5f48 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,6 +5,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Calibration Plugins](./stage-a.md) — Teensy-driven Stage-A bench stack: `stage-a-io` shared I/O, commissioning monitor, and the A1 minimum-depth Bode sweep. +- [Stage-A Function Generator](./stage-a-funcgen.md) — familiarisation plugin: manual sine/square/sawtooth drive with photodiode-measured contrast, firmware-faithful mock, and the reserved waveform-drive protocol fields. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-funcgen.md b/docs/features/stage-a-funcgen.md new file mode 100644 index 0000000..3f70170 --- /dev/null +++ b/docs/features/stage-a-funcgen.md @@ -0,0 +1,57 @@ +# Stage-A Function Generator (`stage-a-funcgen`) + +> Feature brief — familiarisation plugin for the Stage-A bench. +> Protocol source of truth: +> `stage-a-controller/docs/features/waveform-drive.md` (reserved v2 fields). + +## Purpose + +Manual Pockels-cell drive control for getting to know the setup: waveform +(sine / square / sawtooth), frequency, and DAC modulation depth, with the +resulting optical amplitude always **measured** from the photodiode as +`a = ln(V_max/V_min)` — the Pockels V→T response is non-linear, so the DAC +excursion never doubles as a light level. + +## Included + +- `plugins/stage-a-funcgen` crate (`augur-plugin-stage-a-funcgen`): connect / + apply / stop actions, live photodiode waveform view, status table with the + measured contrast, clipping, and stream integrity; +- **`mock` port** (default): the waveform-extended mock controller runs on an + in-process thread and streams a synthetic photodiode response through a + Pockels-like sin² transfer — the complete control loop with zero hardware; +- feature detection against real firmware: 0.2.0 rejects the reserved drive + fields with `unknown_config_field`, which the plugin reports as "no + waveform backend" instead of a fault; +- same fail-closed safety model as `stage-a-monitor` (`LiveCapture` + + `effects_allowed` only; drive parameters are settings, applying them is an + explicit action; local bounds check before any command is sent). + +## Firmware-faithful mock (stage-a-io) + +Delivered together with this plugin, `stage-a-io`'s `MockController` now +mirrors firmware 0.2.0 exactly — verbs, state machine (`SAFE_IDLE` → +`CONFIGURED` → `RUNNING`), error codes/details, single-entry idempotent reply +cache, and unknown-CONFIG-field rejection. The previous mock accepted verbs +and fields the device does not speak (`ARM`, `RUN`, `capabilities=`, +`BAD_*`), which let host bugs pass tests: the A1 sweep reconfigured while +RUNNING (now fixed with STOP-before-CONFIG) and watchdog `!FAULT` notices +were invisible outside an in-flight request (now surfaced as async events by +`StageAClient` and handled by all three plugins). + +## Verification + +`cargo test -p stage-a-io -p augur-plugin-stage-a-funcgen +-p augur-plugin-stage-a-monitor -p augur-plugin-stage-a-a1`: mock +state-machine/error fidelity against `main.cpp`, v1 rejection of waveform +fields, drive-bounds validation, nonlinear sin² contrast response, watchdog +fault propagation, and the full mock round trip (connect → apply sine / +square / saw → measured `a` → stop → reconfigure while driving). + +## Known gaps + +- Real firmware cannot emit a waveform yet; the `waveform-drive.md` fields + stay host+mock-only until the hardware freeze resolves the DAC channel and + safe HVA window. +- No PDQ/sidecar recording in this plugin — it is a familiarisation tool; + evidence-grade recording stays with `stage-a-monitor`/`stage-a-a1`. diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md index 7924c6b..98354e8 100644 --- a/docs/features/stage-a.md +++ b/docs/features/stage-a.md @@ -11,6 +11,7 @@ AugurRs generic host (camera, RAW, EXT_TRIGGER delivery, execution context — ABI v5) │ ├── stage-a-monitor — commissioning: live photodiode view, manual control + ├── stage-a-funcgen — familiarisation: manual waveform drive (see stage-a-funcgen.md) └── stage-a-a1 — A1 minimum-depth a_min(f) sweep │ (exactly one armed plugin owns the device) ▼ @@ -24,8 +25,9 @@ lives entirely in these removable plugins (ADR 005). | Crate | Role | |---|---| -| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, mock controller | +| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, firmware-faithful mock controller (0.2.0 surface + opt-in v2 waveform extension) | | `plugins/stage-a-monitor` | Live decimated waveform, live `a`, integrity status, gated manual CONFIG/START/STOP + expert drive modal | +| `plugins/stage-a-funcgen` | Manual waveform drive (sine/square/saw, frequency, DAC depth) with photodiode-measured `a`; in-process mock port for hardware-free familiarisation | | `plugins/stage-a-a1` | Phase-locked detection (Rayleigh), hardware/software cycle fiducials, bisection + grid sweep, probit `a_min` fit with CI, hot-pixel mask, PDQ + sidecar + results export | ## Safety model @@ -65,7 +67,10 @@ dataset/schema consistency. ## Known gaps - Final Teensy DDS/DAC firmware is blocked on the hardware freeze; the - sweep runs against the v1 protocol and the mock meanwhile. + sweep and the function generator run against the reserved waveform-drive + protocol (`stage-a-controller/docs/features/waveform-drive.md`) and the + waveform-extended mock meanwhile — firmware 0.2.0 rejects the drive + fields with `unknown_config_field` (feature detection). - Marker cycles are protocol-reserved but not yet emitted (`stage-a-controller/docs/features/a1-marker-cycles.md`). - `stage-a-a2` / `stage-a-a3` plugins are not yet implemented; A2 diff --git a/plugins/stage-a-funcgen/Cargo.toml b/plugins/stage-a-funcgen/Cargo.toml new file mode 100644 index 0000000..ffb540e --- /dev/null +++ b/plugins/stage-a-funcgen/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "augur-plugin-stage-a-funcgen" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A function generator: manual waveform/frequency/amplitude drive control with photodiode-measured optical contrast." + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde_json.workspace = true +stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-funcgen/README.md b/plugins/stage-a-funcgen/README.md new file mode 100644 index 0000000..fb57f6c --- /dev/null +++ b/plugins/stage-a-funcgen/README.md @@ -0,0 +1,47 @@ +# Stage-A Function Generator + +Manual control of the Stage-A Pockels-cell drive for familiarisation with the +bench: pick a waveform (**sine**, **square**, **sawtooth**), a frequency, and a +DAC modulation depth, hit *Apply drive*, and watch the photodiode respond live. + +## Why the amplitude is "measured", not set + +`amplitude_dac` commands the *phase*-modulation depth of the Pockels cell. The +cell's voltage→transmission response is non-linear (≈ sin²), so the same DAC +excursion produces different optical amplitudes at different working points. +The plugin therefore always reports the **measured** optical log-contrast + +``` +a = ln(V_max / V_min) (dark-corrected photodiode voltages) +``` + +computed by `stage-a-io`'s calibrated, clipping-guarded estimator — never a +value inferred from the commanded DAC codes. + +## Ports + +| Port | Behaviour | +|---|---| +| `mock` (default) | Runs the waveform-extended mock controller in-process: full command round trip, synthetic photodiode stream through a Pockels-like sin² transfer. Zero hardware, zero risk. | +| `auto` / explicit device | Real Teensy over USB serial. Firmware 0.2.0 has **no waveform backend** and rejects the drive fields (`unknown_config_field`); the plugin reports this clearly. Real drive control needs the future v2 DDS firmware (`stage-a-controller/docs/features/waveform-drive.md`), which is blocked on the hardware freeze. | + +## Views and actions + +- **FuncGen photodiode** — live decimated waveform (volts vs. ms). +- **Function generator** status table — state, firmware, waveform-backend + capability, commanded drive, measured `a`, clipping, stream integrity. +- Actions on the status table: *Connect*, *Disconnect*, *Apply drive*, + *Stop drive*. + +## Safety model + +Same contract as `stage-a-monitor`: + +- serial/mock connections open only while the execution context is + `LiveCapture` with effects allowed — replay can never drive hardware; +- waveform/frequency/amplitude are persistent *settings*, but nothing reaches + the controller until the explicit *Apply drive* **action**; +- drives whose `center ± amplitude` leave the 0–4095 DAC range are refused + locally before any command is sent; +- `process_frame()` only drains the bounded I/O worker queues; +- watchdog `!FAULT` notices from the controller are surfaced immediately. diff --git a/plugins/stage-a-funcgen/plugin.toml b/plugins/stage-a-funcgen/plugin.toml new file mode 100644 index 0000000..df29231 --- /dev/null +++ b/plugins/stage-a-funcgen/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Function Generator" +version = "0.2.0" +description = "Manual Pockels-cell drive control (sine/square/sawtooth, frequency, DAC amplitude) with the resulting optical contrast always measured from the photodiode." +domain = "stage-a" +library = "augur_plugin_stage_a_funcgen" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-funcgen/src/lib.rs b/plugins/stage-a-funcgen/src/lib.rs new file mode 100644 index 0000000..694a21c --- /dev/null +++ b/plugins/stage-a-funcgen/src/lib.rs @@ -0,0 +1,1011 @@ +//! Stage-A function generator — familiarisation plugin. +//! +//! Manual control of the Pockels-cell drive: waveform (sine, square, +//! sawtooth), frequency, and the commanded DAC modulation depth +//! (`amplitude_dac`). The commanded amplitude sets the *phase* modulation +//! of the Pockels cell, which maps non-linearly to transmitted intensity — +//! so the optical amplitude shown here is always the photodiode-measured +//! log-contrast `a = ln(V_max/V_min)`, never the DAC excursion. +//! +//! Firmware 0.2.0 has no waveform backend yet: it rejects the reserved v2 +//! drive fields with `unknown_config_field` (the feature-detection +//! contract, `stage-a-controller/docs/features/waveform-drive.md`). Until +//! the DDS firmware lands, select the **`mock`** port: it runs the +//! waveform-extended mock controller in-process and streams a synthetic +//! photodiode response through a Pockels-like sin² transfer — the full +//! control loop with zero hardware and zero risk. +//! +//! Safety contract (same as `stage-a-monitor`): +//! - devices open only when the execution context is `LiveCapture` with +//! `effects_allowed`; anything else tears the connection down; +//! - drive parameters are persistent *settings*, but nothing starts the +//! hardware except an explicit Apply **action**; +//! - `process_frame()` only drains the bounded I/O worker queues. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, + MockController, MockState, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, +}; + +const WAVEFORM_DATASET_ID: &str = "stage-a-funcgen.waveform"; +const STATUS_DATASET_ID: &str = "stage-a-funcgen.status"; +const WAVEFORM_VIEW_ID: &str = "stage-a-funcgen.waveform.view"; +const STATUS_VIEW_ID: &str = "stage-a-funcgen.status.view"; + +const ACTION_CONNECT: &str = "stage-a-funcgen.connect"; +const ACTION_DISCONNECT: &str = "stage-a-funcgen.disconnect"; +const ACTION_APPLY: &str = "stage-a-funcgen.apply"; +const ACTION_STOP: &str = "stage-a-funcgen.stop"; + +/// Retained sample window for the live view + contrast estimate. +const SAMPLE_RING_CAPACITY: usize = 32_768; +/// Points published per waveform refresh (decimated). +const WAVEFORM_POINTS: usize = 1_024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ConnectionState { + Disconnected, + Connected, + Driving, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Wave { + Sine, + Square, + Saw, +} + +impl Wave { + const VARIANTS: [Wave; 3] = [Wave::Sine, Wave::Square, Wave::Saw]; + + fn name(self) -> &'static str { + match self { + Self::Sine => "SINE", + Self::Square => "SQUARE", + Self::Saw => "SAW", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|w| w.name() == name) + } +} + +/// In-process mock controller thread behind the `mock` port. +struct MockService { + stop: Arc, + join: Option>, +} + +impl MockService { + fn spawn() -> (Self, StageAClient) { + let link = stage_a_io::MockLink::new(); + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); + let join = std::thread::Builder::new() + .name("stage-a-funcgen-mock".into()) + .spawn(move || { + let mut last_block = Instant::now(); + while !thread_stop.load(Ordering::Relaxed) { + controller.poll_commands(); + if controller.state() == MockState::Running + && last_block.elapsed() >= controller.block_period() + { + last_block = Instant::now(); + controller.emit_configured_block(); + } + std::thread::sleep(Duration::from_millis(1)); + } + }) + .expect("spawning the mock controller thread must succeed"); + ( + Self { + stop, + join: Some(join), + }, + StageAClient::new(link.host_end()), + ) + } +} + +impl Drop for MockService { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +pub struct StageAFuncGenPlugin { + enabled: bool, + // -- device -- + worker: Option, + mock_service: Option, + connection: ConnectionState, + firmware: String, + has_waveform_backend: Option, + next_tag: u64, + in_flight: BTreeMap, + last_error: Option, + integrity: StreamIntegrity, + effects_blocked_reason: Option, + // -- settings (drive parameters; applying them is an explicit action) -- + port_hint: String, + wave: Wave, + frequency_hz: f64, + center_dac: i64, + amplitude_dac: i64, + sample_rate_hz: i64, + calibration: AdcCalibration, + // -- data -- + sample_ring: Vec, + ring_next_sample_index: u64, + sample_rate_seen_hz: u32, + contrast: Option, + contrast_error: Option, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAFuncGenPlugin { + fn default() -> Self { + Self { + enabled: false, + worker: None, + mock_service: None, + connection: ConnectionState::Disconnected, + firmware: String::new(), + has_waveform_backend: None, + next_tag: 1, + in_flight: BTreeMap::new(), + last_error: None, + integrity: StreamIntegrity::default(), + effects_blocked_reason: None, + port_hint: "mock".into(), + wave: Wave::Sine, + frequency_hz: 1_000.0, + center_dac: 2_048, + amplitude_dac: 512, + sample_rate_hz: 20_000, + calibration: AdcCalibration::default(), + sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), + ring_next_sample_index: 0, + sample_rate_seen_hz: 0, + contrast: None, + contrast_error: None, + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAFuncGenPlugin { + fn bump_generation(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn connect(&mut self) { + if self.worker.is_some() { + return; + } + if self.port_hint == "mock" { + let (service, client) = MockService::spawn(); + self.mock_service = Some(service); + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + } else { + match open_serial(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + } + Err(err) => self.last_error = Some(err), + } + } + self.bump_generation(); + } + + fn disconnect(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + // Shut the worker down first: its final STOP still needs the + // mock service (if any) alive to be acknowledged. + worker.shutdown(reason); + } + self.mock_service = None; + self.connection = ConnectionState::Disconnected; + self.firmware.clear(); + self.has_waveform_backend = None; + self.in_flight.clear(); + self.bump_generation(); + } + + /// STOP → CONFIG (drive fields) → START, honouring the firmware state + /// machine (CONFIG is only legal from SAFE_IDLE/CONFIGURED). + fn apply_drive(&mut self) { + let center = self.center_dac.clamp(0, 4_095); + let amplitude = self.amplitude_dac.clamp(0, 2_047); + if center + amplitude > 4_095 || amplitude > center { + self.last_error = Some(format!( + "drive: center {center} ± amplitude {amplitude} exceeds the 0–4095 DAC range" + )); + return; + } + let freq_mhz = ((self.frequency_hz.max(0.001)) * 1_000.0).round() as i64; + self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); + self.queue_command( + "drive", + Command::new("CONFIG") + .field("mode", "A1") + .field("wave", self.wave.name()) + .field("freq_mhz", freq_mhz) + .field("center_dac", center) + .field("amplitude_dac", amplitude) + .field("rate_hz", self.sample_rate_hz) + .field("block_samples", 256) + .field("raw", 1) + .field("summary", 1), + ); + self.queue_command("start", Command::new("START")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(true)); + } + } + + fn stop_drive(&mut self) { + self.queue_command("stop", Command::new("STOP").field("reason", "operator")); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + if outputs.is_empty() { + return; + } + let mut changed = false; + let mut stopped: Option = None; + for output in outputs { + changed = true; + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) if err.contains("unknown_config_field") => { + self.has_waveform_backend = Some(false); + self.last_error = Some( + "firmware has no waveform backend (v1) — select the mock port \ + or wait for the v2 DDS firmware" + .into(), + ); + } + Err(err) => { + self.last_error = Some(format!("{purpose}: {err}")); + } + } + } + WorkerOutput::Event(DeviceEvent::Data(frame)) => { + if frame.header.frame_type == FrameType::SamplesU16 { + if let Some(codes) = frame.samples() { + self.sample_rate_seen_hz = frame.header.sample_rate_hz; + self.push_samples(&codes, frame.header.first_sample_index); + } + } + } + WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { + if name == "FAULT" { + if self.connection == ConnectionState::Driving { + self.connection = ConnectionState::Connected; + } + self.last_error = Some(format!( + "controller fault: {} — dropped to SAFE_IDLE", + fields.get("code").map(String::as_str).unwrap_or("unknown") + )); + if let Some(worker) = &self.worker { + let _ = worker.try_send(WorkerRequest::SetPinging(false)); + } + } + } + WorkerOutput::Integrity(integrity) => { + self.integrity = integrity; + } + WorkerOutput::Stopped { reason } => { + stopped = Some(reason); + } + } + } + if let Some(reason) = stopped { + self.worker = None; + self.mock_service = None; + self.connection = ConnectionState::Disconnected; + self.last_error = Some(format!("device connection ended: {reason}")); + } + if changed { + self.refresh_contrast(); + self.bump_generation(); + } + } + + fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { + match purpose { + "hello" => { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + self.has_waveform_backend = Some( + fields + .get("capabilities") + .is_some_and(|caps| caps.split(',').any(|c| c == "WAVE")), + ); + self.connection = ConnectionState::Connected; + } + "drive" => { + self.has_waveform_backend = Some(true); + } + "start" => { + self.connection = ConnectionState::Driving; + } + "stop" => { + if self.connection == ConnectionState::Driving { + self.connection = ConnectionState::Connected; + } + } + _ => {} + } + } + + fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { + self.ring_next_sample_index = first_sample_index + codes.len() as u64; + self.sample_ring.extend_from_slice(codes); + let len = self.sample_ring.len(); + if len > SAMPLE_RING_CAPACITY { + self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); + } + } + + fn refresh_contrast(&mut self) { + if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { + return; + } + match estimate_contrast(&self.sample_ring, &self.calibration) { + Ok(estimate) => { + self.contrast = Some(estimate); + self.contrast_error = None; + } + Err(err) => { + self.contrast = None; + self.contrast_error = Some(err.to_string()); + } + } + } + + fn waveform_dataset(&self) -> Series1dV1 { + let rate = if self.sample_rate_seen_hz > 0 { + f64::from(self.sample_rate_seen_hz) + } else { + self.sample_rate_hz as f64 + }; + let n = self.sample_ring.len(); + let stride = (n / WAVEFORM_POINTS).max(1); + let first_index = self.ring_next_sample_index.saturating_sub(n as u64); + let points: Vec = self + .sample_ring + .iter() + .enumerate() + .step_by(stride) + .map(|(i, &code)| Series1dPoint { + x: (first_index + i as u64) as f64 / rate * 1_000.0, + y: self.calibration.code_to_volts(code), + }) + .collect(); + Series1dV1 { + x_label: "time [ms]".into(), + y_label: "photodiode [V]".into(), + lines: vec![Series1dLine { + name: "photodiode".into(), + points, + }], + } + } + + fn drive_summary(&self) -> String { + format!( + "{} @ {:.3} Hz, {} ± {} DAC", + self.wave.name(), + self.frequency_hz, + self.center_dac, + self.amplitude_dac + ) + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.connection) { + (Some(reason), _) => format!("locked ({reason})"), + (None, ConnectionState::Disconnected) => "disconnected".into(), + (None, ConnectionState::Connected) => "connected".into(), + (None, ConnectionState::Driving) => "driving".into(), + }; + let backend = match self.has_waveform_backend { + Some(true) => "waveform-capable".into(), + Some(false) => "no waveform backend (v1)".into(), + None => "—".into(), + }; + let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { + (Some(estimate), _) => ( + format!("{:.4}", estimate.a), + format!( + "{:.2}% low / {:.2}% high", + estimate.low_clip_fraction * 100.0, + estimate.high_clip_fraction * 100.0 + ), + ), + (None, Some(err)) => ("invalid".into(), err.clone()), + (None, None) => ("—".into(), "—".into()), + }; + let integrity = if self.integrity.is_clean() { + "clean".to_owned() + } else { + format!( + "crc={} gaps={} skipped={} overruns={}", + self.integrity.crc_failures, + self.integrity.sequence_gaps, + self.integrity.skipped_bytes, + self.integrity.dropped_samples + ) + }; + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("firmware", self.firmware.clone()), + text_column("backend", backend), + text_column("drive", self.drive_summary()), + text_column("a", a_text), + text_column("clipping", clip_text), + text_column("integrity", integrity), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("firmware", "Firmware"), + column("backend", "Waveform backend"), + column("drive", "Commanded drive"), + column("a", "Measured a = ln(Vmax/Vmin)"), + column("clipping", "Clipping"), + column("integrity", "Stream integrity"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-funcgen.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } +} + +fn open_serial(port_hint: &str) -> Result, String> { + let path = if port_hint == "auto" { + serial_ports() + .into_iter() + .next() + .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? + } else { + port_hint.to_owned() + }; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +fn serial_ports() -> Vec { + stage_a_io::transport::available_port_names() + .into_iter() + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() +} + +impl Plugin for StageAFuncGenPlugin { + fn name(&self) -> &'static str { + "Stage-A Function Generator" + } + + fn description(&self) -> &'static str { + "Manual Pockels-cell drive (sine/square/sawtooth, frequency, DAC amplitude) with photodiode-measured optical contrast; mock port for hardware-free familiarisation." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect("plugin disabled"); + } + } + + fn reset(&mut self) { + self.sample_ring.clear(); + self.contrast = None; + self.contrast_error = None; + self.bump_generation(); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Fail closed: any pass without live-capture effects tears the + // connection down and refuses commands — even for the mock port, + // so switching the port setting can never bypass the gate. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.worker.is_some() { + self.disconnect("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for action_id in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect("operator"), + ACTION_APPLY => self.apply_drive(), + ACTION_STOP => self.stop_drive(), + _ => {} + } + } + + self.drain_worker(); + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + let wave_variants: Vec = + Wave::VARIANTS.iter().map(|w| w.name().to_owned()).collect(); + let wave_default = Wave::VARIANTS + .iter() + .position(|w| *w == self.wave) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Function generator".into(), + description: Some( + "Drive parameters are settings; nothing reaches the hardware until the \ + Apply action. The optical amplitude is measured from the photodiode — \ + the DAC amplitude is a phase-modulation depth, not a light level." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "mock = in-process simulated controller (no hardware); \ + auto = first Teensy USB serial device" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "wave".into(), + label: "Waveform".into(), + tooltip: Some("SINE, SQUARE, or SAW (sawtooth / Sägezahn)".into()), + kind: SettingKind::Enum { + variants: wave_variants, + default: wave_default, + }, + }, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Drive frequency (sent as integer millihertz)".into()), + kind: SettingKind::F64Drag { + min: 0.001, + max: 200_000.0, + speed: 1.0, + default: self.frequency_hz, + }, + }, + SettingItem { + key: "center_dac".into(), + label: "Center DAC code".into(), + tooltip: Some("Working-point code (0–4095)".into()), + kind: SettingKind::I64Drag { + min: 0, + max: 4_095, + default: self.center_dac, + }, + }, + SettingItem { + key: "amplitude_dac".into(), + label: "Amplitude DAC code".into(), + tooltip: Some( + "Pockels phase-modulation depth (0–2047); the optical contrast \ + this produces is read from the measured a" + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: 2_047, + default: self.amplitude_dac, + }, + }, + SettingItem { + key: "sample_rate_hz".into(), + label: "ADC sample rate".into(), + tooltip: Some("Photodiode sample rate for the feedback stream".into()), + kind: SettingKind::I64Slider { + min: 1_000, + max: 100_000, + default: self.sample_rate_hz, + suffix: Some(" Hz".into()), + }, + }, + SettingItem { + key: "dark_millivolts".into(), + label: "Dark level".into(), + tooltip: Some( + "Light-blocked photodiode level; a is computed from dark-corrected \ + voltages" + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 3_300.0, + speed: 1.0, + default: self.calibration.dark_volts * 1_000.0, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "wave" => Some(json!(self.wave.name())), + "frequency_hz" => Some(json!(self.frequency_hz)), + "center_dac" => Some(json!(self.center_dac)), + "amplitude_dac" => Some(json!(self.amplitude_dac)), + "sample_rate_hz" => Some(json!(self.sample_rate_hz)), + "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "wave" => { + let name = value.as_str().ok_or("wave must be a string")?; + self.wave = Wave::from_name(name) + .ok_or_else(|| format!("unknown waveform: {name} (SINE/SQUARE/SAW)"))?; + Ok(()) + } + "frequency_hz" => { + let hz = value.as_f64().ok_or("frequency_hz must be a number")?; + self.frequency_hz = hz.clamp(0.001, 200_000.0); + Ok(()) + } + "center_dac" => { + self.center_dac = value + .as_i64() + .ok_or("center_dac must be an integer")? + .clamp(0, 4_095); + Ok(()) + } + "amplitude_dac" => { + self.amplitude_dac = value + .as_i64() + .ok_or("amplitude_dac must be an integer")? + .clamp(0, 2_047); + Ok(()) + } + "sample_rate_hz" => { + self.sample_rate_hz = value + .as_i64() + .ok_or("sample_rate_hz must be an integer")? + .clamp(1_000, 100_000); + Ok(()) + } + "dark_millivolts" => { + let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; + self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + entries.push(StatusEntry::Text(match self.connection { + ConnectionState::Disconnected => "FuncGen: disconnected".into(), + ConnectionState::Connected => format!("FuncGen: connected ({})", self.firmware), + ConnectionState::Driving => format!("FuncGen: driving {}", self.drive_summary()), + })); + if let Some(estimate) = &self.contrast { + entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + let dataset_action = |id: &str, title: &str| HostActionDescriptor { + id: id.into(), + title: title.into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }; + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: WAVEFORM_DATASET_ID.into(), + title: "FuncGen photodiode waveform".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No photodiode samples yet — connect and apply a drive.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Function generator status".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Function generator idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: WAVEFORM_VIEW_ID.into(), + title: "FuncGen photodiode".into(), + dataset_id: WAVEFORM_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Function generator".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + dataset_action(ACTION_CONNECT, "Connect"), + dataset_action(ACTION_DISCONNECT, "Disconnect"), + dataset_action(ACTION_APPLY, "Apply drive"), + dataset_action(ACTION_STOP, "Stop drive"), + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), + _ => 0, + } + } +} + +impl Drop for StageAFuncGenPlugin { + fn drop(&mut self) { + self.disconnect("plugin destroyed"); + } +} + +export_plugin!(StageAFuncGenPlugin); + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + fn drain_until bool>( + plugin: &mut StageAFuncGenPlugin, + timeout: Duration, + mut done: F, + ) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + plugin.drain_worker(); + if done(plugin) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("condition not reached within {timeout:?}"); + } + + /// Full mock loop: connect → apply sine → measured a appears → stop. + #[test] + fn mock_port_round_trip_measures_optical_contrast() { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.calibration.dark_volts = 40.0 * 3.3 / 4_095.0; + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + assert_eq!(plugin.has_waveform_backend, Some(true)); + assert_eq!(plugin.firmware, "0.2.0-mock"); + + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving && p.contrast.is_some() + }); + let a = plugin.contrast.as_ref().expect("contrast measured").a; + assert!(a > 0.0, "modulated drive must produce positive contrast"); + assert!(plugin.integrity.is_clean()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + + plugin.stop_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + plugin.disconnect("test done"); + assert_eq!(plugin.connection, ConnectionState::Disconnected); + } + + /// Square and sawtooth are accepted and produce a measurable contrast. + #[test] + fn square_and_saw_waveforms_drive_the_mock() { + for wave in [Wave::Square, Wave::Saw] { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.wave = wave; + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving && p.contrast.is_some() + }); + assert!(plugin.contrast.as_ref().unwrap().a > 0.0); + plugin.disconnect("done"); + } + } + + /// Drives exceeding the DAC range are refused locally, before any + /// command reaches a controller. + #[test] + fn out_of_range_drive_is_rejected_locally() { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.center_dac = 3_000; + plugin.amplitude_dac = 2_000; + plugin.apply_drive(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("exceeds the 0–4095 DAC range"))); + } + + /// Re-applying while driving must STOP first (firmware state machine). + #[test] + fn reapply_while_driving_reconfigures_cleanly() { + let mut plugin = StageAFuncGenPlugin::default(); + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Connected + }); + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving + }); + plugin.frequency_hz = 2_000.0; + plugin.apply_drive(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.connection == ConnectionState::Driving && p.last_error.is_none() + }); + plugin.disconnect("done"); + } +} From 525cc4d5790f2ce96798a47889619eb104b64198 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 15 Jul 2026 19:56:13 +0200 Subject: [PATCH 10/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20replace=20co?= =?UTF-8?q?mmissioning=20plugins=20with=20minimal=20modulation=20and=20pho?= =?UTF-8?q?todiode=20pair?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - delete stage-a-monitor, stage-a-funcgen, stage-a-a1 (too complex for the current bench stage; retained in git history) - add stage-a-modulation: capped power slider, CONST/SINE/SQUARE with frequency and min threshold, immediate MOD transfer, board-reported DAC code - add stage-a-photodiode: SMA5/pin18/A4 stream readout on the second CDC port with RAW and EXCITATION (I_exc = I_tot - I_pd) modes and rolling chart - extend the stage-a-io mock to firmware 0.3.0 (MOD verb, capabilities) - ADR 006 (two plugins, one port each), rewritten stage-a brief, doc updates --- Cargo.toml | 6 +- docs/adr/006-stage-a-two-plugin-split.md | 48 + docs/features/README.md | 5 +- docs/features/stage-a-funcgen.md | 57 - docs/features/stage-a-modulation.md | 37 + docs/features/stage-a-photodiode.md | 34 + docs/features/stage-a.md | 94 +- plugins/stage-a-a1/README.md | 66 - plugins/stage-a-a1/plugin.toml | 7 - plugins/stage-a-a1/src/analysis.rs | 610 -------- plugins/stage-a-a1/src/lib.rs | 1254 ----------------- plugins/stage-a-a1/src/sweep.rs | 368 ----- plugins/stage-a-funcgen/README.md | 47 - plugins/stage-a-funcgen/plugin.toml | 7 - plugins/stage-a-funcgen/src/lib.rs | 1011 ------------- .../Cargo.toml | 4 +- plugins/stage-a-modulation/README.md | 33 + plugins/stage-a-modulation/plugin.toml | 7 + plugins/stage-a-modulation/src/lib.rs | 844 +++++++++++ plugins/stage-a-monitor/Cargo.toml | 15 - plugins/stage-a-monitor/README.md | 39 - plugins/stage-a-monitor/plugin.toml | 7 - plugins/stage-a-monitor/src/lib.rs | 831 ----------- .../Cargo.toml | 6 +- plugins/stage-a-photodiode/README.md | 25 + plugins/stage-a-photodiode/plugin.toml | 7 + plugins/stage-a-photodiode/src/lib.rs | 774 ++++++++++ stage-a-io/src/lib.rs | 5 +- stage-a-io/src/mock.rs | 172 ++- 29 files changed, 2006 insertions(+), 4414 deletions(-) create mode 100644 docs/adr/006-stage-a-two-plugin-split.md delete mode 100644 docs/features/stage-a-funcgen.md create mode 100644 docs/features/stage-a-modulation.md create mode 100644 docs/features/stage-a-photodiode.md delete mode 100644 plugins/stage-a-a1/README.md delete mode 100644 plugins/stage-a-a1/plugin.toml delete mode 100644 plugins/stage-a-a1/src/analysis.rs delete mode 100644 plugins/stage-a-a1/src/lib.rs delete mode 100644 plugins/stage-a-a1/src/sweep.rs delete mode 100644 plugins/stage-a-funcgen/README.md delete mode 100644 plugins/stage-a-funcgen/plugin.toml delete mode 100644 plugins/stage-a-funcgen/src/lib.rs rename plugins/{stage-a-funcgen => stage-a-modulation}/Cargo.toml (58%) create mode 100644 plugins/stage-a-modulation/README.md create mode 100644 plugins/stage-a-modulation/plugin.toml create mode 100644 plugins/stage-a-modulation/src/lib.rs delete mode 100644 plugins/stage-a-monitor/Cargo.toml delete mode 100644 plugins/stage-a-monitor/README.md delete mode 100644 plugins/stage-a-monitor/plugin.toml delete mode 100644 plugins/stage-a-monitor/src/lib.rs rename plugins/{stage-a-a1 => stage-a-photodiode}/Cargo.toml (51%) create mode 100644 plugins/stage-a-photodiode/README.md create mode 100644 plugins/stage-a-photodiode/plugin.toml create mode 100644 plugins/stage-a-photodiode/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 3c96d7b..dbee199 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,8 @@ [workspace] members = [ "stage-a-io", - "plugins/stage-a-monitor", - "plugins/stage-a-a1", - "plugins/stage-a-funcgen", + "plugins/stage-a-modulation", + "plugins/stage-a-photodiode", "plugins/localization", "plugins/reconstruction", "plugins/focus-metrics", @@ -29,3 +28,4 @@ egui = "0.27" rustfft = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" +serialport = "4" diff --git a/docs/adr/006-stage-a-two-plugin-split.md b/docs/adr/006-stage-a-two-plugin-split.md new file mode 100644 index 0000000..99971e3 --- /dev/null +++ b/docs/adr/006-stage-a-two-plugin-split.md @@ -0,0 +1,48 @@ +# ADR 006 — Stage-A simplification: two plugins, one serial port each + +- **Status:** Accepted +- **Date:** 2026-07-15 +- **Amends:** ADR 005 (Stage-A device ownership) + +## Context + +The three commissioning plugins (`stage-a-monitor`, `stage-a-funcgen`, `stage-a-a1`, ~3100 lines) +bundled experiment state machines, contrast estimation, and drive control into UIs that were too +complex and opaque for the current bench stage. What the bench actually needs now is: + +1. direct, immediate control of the laser modulation output (capped power slider, + constant/sine/square with frequency), and +2. a plain readout of the photodiode (raw, or inverted to excitation power). + +Both need the same Teensy, but ADR 005 fixes one owner per serial port — and a context-bus +coupling (one plugin republishing data for the other) would make the readout depend on the +control plugin's connection. + +## Decision + +1. **The firmware enumerates two USB CDC ports** (`USB_DUAL_SERIAL`, `stage-a-controller` + ADR 002): port 1 keeps the v1 command protocol; port 2 free-runs a plain-ASCII photodiode + stream. ADR 005's rule is unchanged — one owner per port — there are simply two ports now. +2. **Two minimal plugins replace the three commissioning plugins** (deleted 2026-07-15, retained + in git history): + - `stage-a-modulation` owns the command port (`docs/features/stage-a-modulation.md`); + - `stage-a-photodiode` owns the stream port (`docs/features/stage-a-photodiode.md`). +3. **`stage-a-io` stays** as the protocol library (wire format, client, worker, firmware-faithful + mock — the mock now models firmware 0.3.0's `MOD` verb). The A1/A2/A3 experiment plugins will + build on it again when the bench reaches that stage; the estimator/pdq/sidecar modules remain + for that purpose even though no current plugin uses them. +4. **Immediate transfer replaces the Apply-action pattern** in `stage-a-modulation`: setting + changes are sent to the device as they happen (the operator's explicit request), still behind + the fail-closed execution-context gate. The firmware output is set-and-hold; the explicit + "Output OFF" action is the only stop. + +## Consequences + +- Each plugin is a few hundred transparent lines with a single concern; the photodiode plugin + does not even depend on `stage-a-io`. +- Both plugins work independently — either can connect, disconnect, or crash without affecting + the other. +- Wire-protocol changes still land firmware-first (`stage-a-controller/include/wire_protocol.h` + and command grammar), then in `stage-a-io`'s client/mock. +- The A1 min-depth workflow is gone from the tree until it is rebuilt on the simplified stack; + its last state is tagged by the deletion commit. diff --git a/docs/features/README.md b/docs/features/README.md index 39e5f48..eb5ffe4 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -4,8 +4,9 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs -- [Stage-A Calibration Plugins](./stage-a.md) — Teensy-driven Stage-A bench stack: `stage-a-io` shared I/O, commissioning monitor, and the A1 minimum-depth Bode sweep. -- [Stage-A Function Generator](./stage-a-funcgen.md) — familiarisation plugin: manual sine/square/sawtooth drive with photodiode-measured contrast, firmware-faithful mock, and the reserved waveform-drive protocol fields. +- [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. +- [Stage-A Modulation](./stage-a-modulation.md) — capped power slider + constant/sine/square laser-modulation drive on the command port, applied immediately. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the stream port: raw values or excitation power `I_exc = I_tot − I_pd`. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-funcgen.md b/docs/features/stage-a-funcgen.md deleted file mode 100644 index 3f70170..0000000 --- a/docs/features/stage-a-funcgen.md +++ /dev/null @@ -1,57 +0,0 @@ -# Stage-A Function Generator (`stage-a-funcgen`) - -> Feature brief — familiarisation plugin for the Stage-A bench. -> Protocol source of truth: -> `stage-a-controller/docs/features/waveform-drive.md` (reserved v2 fields). - -## Purpose - -Manual Pockels-cell drive control for getting to know the setup: waveform -(sine / square / sawtooth), frequency, and DAC modulation depth, with the -resulting optical amplitude always **measured** from the photodiode as -`a = ln(V_max/V_min)` — the Pockels V→T response is non-linear, so the DAC -excursion never doubles as a light level. - -## Included - -- `plugins/stage-a-funcgen` crate (`augur-plugin-stage-a-funcgen`): connect / - apply / stop actions, live photodiode waveform view, status table with the - measured contrast, clipping, and stream integrity; -- **`mock` port** (default): the waveform-extended mock controller runs on an - in-process thread and streams a synthetic photodiode response through a - Pockels-like sin² transfer — the complete control loop with zero hardware; -- feature detection against real firmware: 0.2.0 rejects the reserved drive - fields with `unknown_config_field`, which the plugin reports as "no - waveform backend" instead of a fault; -- same fail-closed safety model as `stage-a-monitor` (`LiveCapture` + - `effects_allowed` only; drive parameters are settings, applying them is an - explicit action; local bounds check before any command is sent). - -## Firmware-faithful mock (stage-a-io) - -Delivered together with this plugin, `stage-a-io`'s `MockController` now -mirrors firmware 0.2.0 exactly — verbs, state machine (`SAFE_IDLE` → -`CONFIGURED` → `RUNNING`), error codes/details, single-entry idempotent reply -cache, and unknown-CONFIG-field rejection. The previous mock accepted verbs -and fields the device does not speak (`ARM`, `RUN`, `capabilities=`, -`BAD_*`), which let host bugs pass tests: the A1 sweep reconfigured while -RUNNING (now fixed with STOP-before-CONFIG) and watchdog `!FAULT` notices -were invisible outside an in-flight request (now surfaced as async events by -`StageAClient` and handled by all three plugins). - -## Verification - -`cargo test -p stage-a-io -p augur-plugin-stage-a-funcgen --p augur-plugin-stage-a-monitor -p augur-plugin-stage-a-a1`: mock -state-machine/error fidelity against `main.cpp`, v1 rejection of waveform -fields, drive-bounds validation, nonlinear sin² contrast response, watchdog -fault propagation, and the full mock round trip (connect → apply sine / -square / saw → measured `a` → stop → reconfigure while driving). - -## Known gaps - -- Real firmware cannot emit a waveform yet; the `waveform-drive.md` fields - stay host+mock-only until the hardware freeze resolves the DAC channel and - safe HVA window. -- No PDQ/sidecar recording in this plugin — it is a familiarisation tool; - evidence-grade recording stays with `stage-a-monitor`/`stage-a-a1`. diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md new file mode 100644 index 0000000..c495d9a --- /dev/null +++ b/docs/features/stage-a-modulation.md @@ -0,0 +1,37 @@ +# Stage-A Modulation + +- **Crate:** `plugins/stage-a-modulation` (`augur-plugin-stage-a-modulation`) +- **Firmware:** `stage-a-controller` 0.3.0+ (`MOD` capability), Teensy **command port** +- **Status:** Active (2026-07-15) — replaces `stage-a-funcgen` and the drive half of + `stage-a-monitor` + +## What it is + +The simplest possible laser-modulation control for the Stage-A bench: one power slider in DAC +codes (J23 output, `DAC1.4`), a mode select (`CONST`/`SINE`/`SQUARE`) with frequency +(0.01–2000 Hz) and a min threshold for the periodic modes, and a user-set **max limit** that caps +the slider so a device with a lower tolerated input voltage can never be overdriven from the UI. + +Every accepted setting change is transferred to the Teensy **immediately** as one `MOD` command — +no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the +board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded values. + +## Contract + +- Owns the Teensy **command port** exclusively (one owner per port, ADR 006). The photodiode + stream port belongs to `stage-a-photodiode`. +- Uses `stage-a-io` (`StageAClient`, `IoWorker`, `Command`) for framing, idempotent retries, and + the bounded background I/O thread; `process_frame()` never blocks on serial. +- Fail-closed effects gate: the connection only exists while the host execution context allows + hardware effects. +- Firmware output is **set-and-hold** (`stage-a-controller` ADR 002): disconnecting does not stop + the modulation. The explicit **Output OFF** action sends `MOD wave=OFF`. +- Safety invariants enforced plugin-side: `level ≤ max_level`, `min_level ≤ level`; the firmware + waveform peaks at `level` by construction. +- `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. + +## Verification + +`cargo test -p augur-plugin-stage-a-modulation` — mock round trips: immediate transfer on slider +change, board-code echo, max-cap clamping (including schema regeneration), square drive with min +threshold, Output OFF. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md new file mode 100644 index 0000000..976b9a8 --- /dev/null +++ b/docs/features/stage-a-photodiode.md @@ -0,0 +1,34 @@ +# Stage-A Photodiode + +- **Crate:** `plugins/stage-a-photodiode` (`augur-plugin-stage-a-photodiode`) +- **Firmware:** `stage-a-controller` 0.3.0+ (`PDSTREAM`), Teensy **stream port** (second CDC port) +- **Status:** Active (2026-07-15) — replaces the readout half of `stage-a-monitor` + +## What it is + +A minimal live readout of the photodiode on **board SMA5 → Teensy pin 18 / A4**. The firmware +streams `PD code= n= t_ms=` lines at 50 Hz on its second USB serial port; a +background thread parses them into a bounded ring, and the plugin shows the newest value plus a +rolling chart (1–120 s window). + +Two modes: + +- **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). +- **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light + removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set + reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. + +## Contract + +- Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the + plugin is read-only by construction and needs no protocol library — it depends only on + `serialport` and parses one line format. +- Same fail-closed effects gate as the other stage-a plugins for consistent device handling. +- Garbage on the port (e.g. the binary command port picked by mistake) parses to nothing and is + bounded — it can neither grow memory nor produce fake values. +- `mock` port synthesizes a slow sine for hardware-free testing. + +## Verification + +`cargo test -p augur-plugin-stage-a-photodiode` — line parsing (including clamping and rejection), +excitation inversion against the reference, mock reader filling ring/series, ring bound. diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md index 98354e8..ab9167f 100644 --- a/docs/features/stage-a.md +++ b/docs/features/stage-a.md @@ -1,77 +1,31 @@ -# Stage-A calibration plugins (`stage-a-io`, `stage-a-monitor`, `stage-a-a1`) +# Stage-A Bench Stack -> Feature brief — first delivery of the Stage-A camera-calibration stack. -> Design source of truth: knowledge base -> `methodology/stage-a-control-software.md` and -> `methodology/camera-calibration.md` (A1 protocol). +- **Status:** Simplified two-plugin setup (2026-07-15, ADR 006) +- **Firmware:** `stage-a-controller` 0.3.0 (Teensy 4.1 on Hermit V2r1, `USB_DUAL_SERIAL`) -## Architecture +## Current shape -```text -AugurRs generic host (camera, RAW, EXT_TRIGGER delivery, execution context — ABI v5) - │ - ├── stage-a-monitor — commissioning: live photodiode view, manual control - ├── stage-a-funcgen — familiarisation: manual waveform drive (see stage-a-funcgen.md) - └── stage-a-a1 — A1 minimum-depth a_min(f) sweep - │ (exactly one armed plugin owns the device) - ▼ - stage-a-io (this repo, plain lib) ── USB serial ── Teensy stage-a-controller -``` +The Teensy enumerates as **two** USB serial ports, and each is owned by exactly one plugin: -AugurRs itself gains no Teensy or serial abstraction — device ownership -lives entirely in these removable plugins (ADR 005). +| Port | Content | Owner | +|---|---|---| +| command port (first) | v1 ASCII commands + PDA1 binary frames | [`stage-a-modulation`](./stage-a-modulation.md) | +| stream port (second) | free-running `PD code=… n=… t_ms=…` lines, 50 Hz | [`stage-a-photodiode`](./stage-a-photodiode.md) | -## Crates +- **`stage-a-modulation`** — capped power slider + constant/sine/square drive of the laser + modulation input (J23), transferred to the Teensy immediately; shows the board-reported DAC + code. Firmware output is set-and-hold; "Output OFF" is the explicit stop. +- **`stage-a-photodiode`** — live readout of SMA5/pin 18/A4, raw or inverted to excitation power + `I_exc = I_tot − I_pd` against a user-set reference. +- **`stage-a-io`** (shared non-plugin library) — PDA1 wire format, typed client with idempotent + retries, bounded I/O worker, and a firmware-faithful mock (including the 0.3.0 `MOD` verb). + The estimator/pdq/sidecar modules are retained for the future A1–A3 experiment plugins. -| Crate | Role | -|---|---| -| `stage-a-io` | PDA1 wire protocol (fragmentation-tolerant, CRC-resyncing parser), v1 ASCII commands with idempotent sequence retries, bounded background I/O worker, `.pdq` writer, JSON run sidecar, calibrated clipping-guarded optical-contrast estimator, firmware-faithful mock controller (0.2.0 surface + opt-in v2 waveform extension) | -| `plugins/stage-a-monitor` | Live decimated waveform, live `a`, integrity status, gated manual CONFIG/START/STOP + expert drive modal | -| `plugins/stage-a-funcgen` | Manual waveform drive (sine/square/saw, frequency, DAC depth) with photodiode-measured `a`; in-process mock port for hardware-free familiarisation | -| `plugins/stage-a-a1` | Phase-locked detection (Rayleigh), hardware/software cycle fiducials, bisection + grid sweep, probit `a_min` fit with CI, hot-pixel mask, PDQ + sidecar + results export | +## History -## Safety model - -- Serial ports open only when `HostContext::execution()` reports - `LiveCapture` **and** `effects_allowed` (host constructs this fail-closed; - only the active live-capture worker qualifies). Replay can never re-arm - hardware, even from a sidecar that contains a runnable setup. -- All hardware commands are host **actions**; persistent settings never - start hardware after a reload. -- Any CRC error, frame-sequence gap, or ADC overrun invalidates the - measurement point; invalid points are re-measured, never patched, and - the run sidecar records the counters. -- The firmware watchdog (1.5 s) drops the controller to `SAFE_IDLE` - independently of host-side cleanup. - -## Statistics (A1) - -Detection is a phase-uniformity test (background activity is uniform in -drive phase; signal is phase-locked), with the frequency-scan multiplicity -Bonferroni-charged when the software clock-skew lock substitutes for the -missing trigger cable. `a_min` is the fitted `N = 0.5` crossing of a -probit in `ln a` with a profile CI — not a raw bisection endpoint — and -`a` is always the photodiode-measured contrast. Details and rationale: -`plugins/stage-a-a1/README.md`. - -## Verification - -`cargo test` (38 tests): wire fragmentation/CRC-resync/overrun, retry -idempotency against the mock controller, worker round-trip + clean STOP, -estimator recovery/clipping/headroom guards, Rayleigh calibration on -uniform and locked phases, background-immunity, clock-skew recovery -(300 ppm), fiducial folding, probit fit recovery, sweep convergence to a -synthetic `a_min`, exhaustion/invalid-window handling, hot-pixel masking, -dataset/schema consistency. - -## Known gaps - -- Final Teensy DDS/DAC firmware is blocked on the hardware freeze; the - sweep and the function generator run against the reserved waveform-drive - protocol (`stage-a-controller/docs/features/waveform-drive.md`) and the - waveform-extended mock meanwhile — firmware 0.2.0 rejects the drive - fields with `unknown_config_field` (feature detection). -- Marker cycles are protocol-reserved but not yet emitted - (`stage-a-controller/docs/features/a1-marker-cycles.md`). -- `stage-a-a2` / `stage-a-a3` plugins are not yet implemented; A2 - additionally requires the physical trigger cable. +The earlier commissioning stack (`stage-a-monitor`, `stage-a-funcgen`, `stage-a-a1` — device +monitor with calibrated contrast, waveform familiarisation, and the A1 minimum-depth Bode sweep) +was removed on 2026-07-15 as too complex for the current bench stage (ADR 006). It remains in git +history; the experiment plugins will be rebuilt on the simplified stack when the bench needs +them. Device-ownership and safety rules: ADR 005 (one owner per port, fail-closed effects gate) +as amended by ADR 006. diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md deleted file mode 100644 index 48055cf..0000000 --- a/plugins/stage-a-a1/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Stage-A A1 — minimum-depth Bode calibration - -Measures `a_min(f)`: the smallest optical log-contrast that still produces -phase-locked camera events, per drive frequency. `|H(f)| = C / a_min(f)`; -the knee of the curve is the pixel bandwidth `f_c(I)`, and the plateau of -`a_min` reads out the contrast quantum `C` (which seeds A3). Protocol -design: knowledge base `methodology/camera-calibration.md` (A1) and -`methodology/stage-a-control-software.md`. - -## How it decides "events just appeared" - -- **Detector — phase, not counts.** Background activity is uniform in - drive phase; modulation events are phase-locked. Each measurement window - is folded and tested with the **Rayleigh test**; background is discounted - automatically instead of subtracting a drifting absolute rate. -- **Cycle fiducial.** With the phase-0 TTL wired into `EXT_TRIGGER`, the - camera-clock edges from `frame.external_triggers()` mark each cycle. - Without the cable, the drive frequency is **refined against the events** - (Rayleigh-power scan over ±ppm around the commanded value — recovers the - Teensy↔camera clock skew); the scan multiplicity is Bonferroni-charged - to the significance threshold. -- **Estimator.** The mean phase-locked events per half-cycle comes from the - positive excess over the median phase-bin occupancy. -- **a_min is a fitted crossing.** The 0→1 step is smeared by shot-noise - first-passage randomness and per-pixel threshold dispersion, so a_min is - the fitted `N = 0.5` crossing of a probit in `ln a`, with a profile - confidence interval. The fitted transition width is a free preview of - the smear (σ_C + FPT). -- **Hot pixels.** An unmodulated reference window at run start builds a - median+5·MAD mask; masked pixels never enter the statistics, and the - mask size is recorded in the sidecar. -- **`a` is measured light.** Every point's contrast comes from the - photodiode ADC through the calibrated, clipping-guarded estimator in - `stage-a-io` — never from the commanded DAC code. Invalid windows - (clipping, CRC/sequence/overrun faults) are re-measured, never patched. - -## Run flow - -`Arm controller` → `Run A1 sweep`: reference window (hot-pixel mask) → -per frequency: bisection on the drive code until the detection boundary is -bracketed → log-spaced grid across the transition → probit fit → -next frequency. Views: `a_min(f)` with CI, live phase histogram, `N(a)` -staircase, run status. Raw PDA1 frames go to -`~/.augur/stage-a-runs/.pdq` with a JSON sidecar and a results -export; final numbers must be recomputed from the camera RAW + PDQ. - -ON and OFF are measured in **separate runs** (settings → Polarity) — the -comparator paths are asymmetric and must never be pooled. - -## Safety - -Fails closed on the ABI v5 execution context exactly like -`stage-a-monitor`: serial I/O only in the active live-capture worker; -Arm/Run/Stop are host actions, never settings; the firmware watchdog -drops to `SAFE_IDLE` independently of host cleanup. - -## Current limitations - -- The Teensy DDS/DAC firmware is still the ADC-only commissioning build — - closed-loop sweeps run against the protocol but the final stimulus - backend is blocked on the hardware freeze (see `stage-a-controller`). -- Marker cycles (periodic full-depth optical anchors) are specced for the - firmware but not yet emitted; the software frequency lock covers the - missing-trigger-cable case meanwhile. -- Measured sample cadence validation and the A5 refractory validity bound - `2fa/C ≪ 1/τ_refr` are recorded, not yet enforced. diff --git a/plugins/stage-a-a1/plugin.toml b/plugins/stage-a-a1/plugin.toml deleted file mode 100644 index 62f6089..0000000 --- a/plugins/stage-a-a1/plugin.toml +++ /dev/null @@ -1,7 +0,0 @@ -name = "Stage-A A1 Min-Depth" -version = "0.2.0" -description = "Event-native Bode calibration: a_min(f) via phase-locked detection, drive bisection, and probit fitting." -domain = "stage-a" -library = "augur_plugin_stage_a_a1" -phase = "raw_events" -min_augur_version = "1.0.0" diff --git a/plugins/stage-a-a1/src/analysis.rs b/plugins/stage-a-a1/src/analysis.rs deleted file mode 100644 index 0c3ce31..0000000 --- a/plugins/stage-a-a1/src/analysis.rs +++ /dev/null @@ -1,610 +0,0 @@ -//! Statistical core of the A1 minimum-depth measurement. -//! -//! ## Why phase, not raw counts -//! -//! Background activity (BA) is uniform in modulation phase; genuine -//! modulation events are phase-locked to the drive. Testing for a -//! phase-locked component (Rayleigh test) therefore discounts uniform -//! background *automatically*, instead of requiring an absolute background -//! rate that drifts with temperature. "Mean events per half-cycle > 1" is -//! kept as the *estimator* (it is the quantity `⌊a·|H|/C⌋` predicts), but -//! the *detector* is the phase test. -//! -//! ## Cycle fiducials without the trigger cable -//! -//! With the phase-0 TTL wired, `frame.external_triggers()` marks each cycle -//! on the camera clock. Without it, the Teensy and camera clocks drift -//! (tens of ppm — folding dies after ~0.1 s at 10 kHz), so the drive -//! frequency is *refined against the events themselves*: scan a small -//! window around the commanded frequency and keep the value maximising the -//! Rayleigh power. The scan multiplicity is charged to the significance -//! test (Bonferroni). -//! -//! ## a_min as a fitted crossing -//! -//! Near threshold the 0→1 step of `⌊a·|H|/C⌋` is smeared by shot-noise -//! first-passage randomness and per-pixel threshold dispersion, so "events -//! just vanish" is not a crisp edge. a_min is defined as the fitted point -//! where the mean phase-locked events per half-cycle crosses 0.5, from a -//! probit-in-ln(a) fit over the transition, with a profile confidence -//! interval. The plateau of a_min(f) reads out the contrast quantum C. - -// --------------------------------------------------------------------------- -// Phase folding -// --------------------------------------------------------------------------- - -/// Folds event timestamps at `frequency_hz` relative to `t0_us`, -/// returning phases in `[0, 1)`. -pub fn fold_phases( - timestamps_us: impl Iterator, - t0_us: u64, - frequency_hz: f64, -) -> Vec { - let period_us = 1.0e6 / frequency_hz; - timestamps_us - .map(|t| { - let dt = t.saturating_sub(t0_us) as f64; - (dt / period_us).fract() - }) - .collect() -} - -/// Folds against explicit cycle-start fiducials (rising trigger edges): -/// each event's phase is its position inside the enclosing cycle. Events -/// before the first or after the last fiducial are dropped (their cycle -/// length is unknown). -pub fn fold_phases_with_fiducials(events: &[u64], cycle_starts_us: &[u64]) -> Vec { - if cycle_starts_us.len() < 2 { - return Vec::new(); - } - let mut phases = Vec::with_capacity(events.len()); - for &t in events { - let idx = match cycle_starts_us.binary_search(&t) { - Ok(i) => i, - Err(0) => continue, - Err(i) => i - 1, - }; - if idx + 1 >= cycle_starts_us.len() { - continue; - } - let start = cycle_starts_us[idx]; - let end = cycle_starts_us[idx + 1]; - if end <= start { - continue; - } - phases.push((t - start) as f64 / (end - start) as f64); - } - phases -} - -// --------------------------------------------------------------------------- -// Rayleigh test -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct RayleighResult { - pub n: usize, - /// Resultant length in [0, 1]. - pub r: f64, - /// Z = n·R². - pub z: f64, - /// Approximate p-value under uniformity, `exp(-Z)` with the standard - /// small-sample correction (Zar / Wilkie). - pub p_value: f64, -} - -pub fn rayleigh_test(phases: &[f64]) -> RayleighResult { - let n = phases.len(); - if n == 0 { - return RayleighResult { - n, - r: 0.0, - z: 0.0, - p_value: 1.0, - }; - } - let (mut c, mut s) = (0.0_f64, 0.0_f64); - for &phase in phases { - let angle = 2.0 * std::f64::consts::PI * phase; - c += angle.cos(); - s += angle.sin(); - } - let r = (c * c + s * s).sqrt() / n as f64; - let z = n as f64 * r * r; - let nf = n as f64; - let p = (-z).exp() - * (1.0 + (2.0 * z - z * z) / (4.0 * nf) - - (24.0 * z - 132.0 * z * z + 76.0 * z.powi(3) - 9.0 * z.powi(4)) / (288.0 * nf * nf)); - RayleighResult { - n, - r, - z, - p_value: p.clamp(0.0, 1.0), - } -} - -// --------------------------------------------------------------------------- -// Frequency refinement (clock-skew recovery without a trigger cable) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct FrequencyLock { - pub frequency_hz: f64, - pub rayleigh: RayleighResult, - /// Number of candidate frequencies tested — multiply into the - /// significance threshold (Bonferroni). - pub trials: usize, -} - -/// Scans `±window_ppm` around `nominal_hz` and returns the frequency with -/// the maximum Rayleigh power. The step is chosen so consecutive candidates -/// dephase by ≤ 0.1 cycle over the observation span (finer is wasted). -pub fn refine_frequency( - timestamps_us: &[u64], - nominal_hz: f64, - window_ppm: f64, -) -> Option { - let (&first, &last) = (timestamps_us.first()?, timestamps_us.last()?); - let span_s = (last.saturating_sub(first)) as f64 / 1.0e6; - if span_s <= 0.0 { - return None; - } - let df_step = 0.1 / span_s; - let half_window_hz = nominal_hz * window_ppm * 1e-6; - let steps = ((half_window_hz / df_step).ceil() as i64).clamp(0, 5_000); - let mut best: Option = None; - let trials = (2 * steps + 1) as usize; - for k in -steps..=steps { - let f = nominal_hz + k as f64 * df_step; - if f <= 0.0 { - continue; - } - let phases = fold_phases(timestamps_us.iter().copied(), first, f); - let stat = rayleigh_test(&phases); - if best.as_ref().is_none_or(|b| stat.z > b.rayleigh.z) { - best = Some(FrequencyLock { - frequency_hz: f, - rayleigh: stat, - trials, - }); - } - } - best -} - -// --------------------------------------------------------------------------- -// Phase-locked excess (the events/half-cycle estimator) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, PartialEq)] -pub struct PhaseHistogram { - pub bins: Vec, - pub total: usize, -} - -pub fn phase_histogram(phases: &[f64], bin_count: usize) -> PhaseHistogram { - let mut bins = vec![0_u32; bin_count.max(1)]; - for &phase in phases { - let idx = ((phase * bins.len() as f64) as usize).min(bins.len() - 1); - bins[idx] += 1; - } - PhaseHistogram { - bins, - total: phases.len(), - } -} - -/// Estimates the phase-locked event count above the uniform background. -/// -/// The per-bin background is the *median* bin occupancy — robust because -/// the locked cluster occupies a minority of bins. Returns the summed -/// positive excess. Dividing by the number of observed cycles gives the -/// mean phase-locked events per cycle (per polarity: one burst per cycle). -pub fn phase_locked_excess(histogram: &PhaseHistogram) -> f64 { - if histogram.bins.is_empty() { - return 0.0; - } - let mut sorted = histogram.bins.clone(); - sorted.sort_unstable(); - let median = f64::from(sorted[sorted.len() / 2]); - histogram - .bins - .iter() - .map(|&count| (f64::from(count) - median).max(0.0)) - .sum() -} - -// --------------------------------------------------------------------------- -// Detection verdict for one (frequency, amplitude) measurement -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct DetectionVerdict { - pub detected: bool, - pub p_value: f64, - /// Bonferroni-corrected significance threshold actually applied. - pub alpha_effective: f64, - /// Mean phase-locked events per cycle (per polarity), background-free. - pub locked_events_per_cycle: f64, -} - -/// Decides whether phase-locked modulation events are present. -/// -/// `alpha` is the per-measurement false-positive budget; `trials` is the -/// look-elsewhere multiplicity (frequency-scan candidates × bisection -/// steps), charged via Bonferroni. -pub fn detect( - rayleigh: RayleighResult, - excess: f64, - observed_cycles: f64, - alpha: f64, - trials: usize, -) -> DetectionVerdict { - let alpha_effective = alpha / trials.max(1) as f64; - DetectionVerdict { - detected: rayleigh.p_value < alpha_effective, - p_value: rayleigh.p_value, - alpha_effective, - locked_events_per_cycle: if observed_cycles > 0.0 { - excess / observed_cycles - } else { - 0.0 - }, - } -} - -// --------------------------------------------------------------------------- -// a_min fit: probit in ln(a) with profile CI -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct MinDepthFit { - /// a at which the mean locked events/half-cycle crosses 0.5. - pub a_min: f64, - /// Profile interval (Δ SSE ≤ SSE_min · (1 + 2/dof)); honest-but-cheap. - pub a_min_low: f64, - pub a_min_high: f64, - /// Transition width in ln(a) — first look at σ_C + FPT smear. - pub sigma_ln_a: f64, - pub points_used: usize, -} - -/// One measured amplitude point for the fit. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct DepthPoint { - /// Measured optical log-contrast (photodiode, never the drive code). - pub a: f64, - /// Mean phase-locked events per half-cycle at this contrast. - pub events_per_half_cycle: f64, -} - -fn standard_normal_cdf(z: f64) -> f64 { - // Abramowitz & Stegun 7.1.26 via erf; |error| < 1.5e-7. - let x = z / std::f64::consts::SQRT_2; - let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); - let poly = t - * (0.254_829_592 - + t * (-0.284_496_736 - + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); - let erf_abs = 1.0 - poly * (-x * x).exp(); - let erf = if x >= 0.0 { erf_abs } else { -erf_abs }; - 0.5 * (1.0 + erf) -} - -/// Fits `N(a) = Φ((ln a − μ)/σ)` over the transition region and reports -/// `a_min = e^μ` (the N = 0.5 crossing). Points far above the first step -/// (`N > 1.5`) are excluded — there the staircase's higher steps dominate -/// and the single-step model no longer applies. -pub fn fit_min_depth(points: &[DepthPoint]) -> Option { - let usable: Vec = points - .iter() - .copied() - .filter(|p| { - p.a > 0.0 && p.events_per_half_cycle.is_finite() && p.events_per_half_cycle <= 1.5 - }) - .collect(); - if usable.len() < 3 { - return None; - } - let has_low = usable.iter().any(|p| p.events_per_half_cycle < 0.4); - let has_high = usable.iter().any(|p| p.events_per_half_cycle > 0.6); - if !has_low || !has_high { - return None; - } - - let ln_min = usable - .iter() - .map(|p| p.a.ln()) - .fold(f64::INFINITY, f64::min); - let ln_max = usable - .iter() - .map(|p| p.a.ln()) - .fold(f64::NEG_INFINITY, f64::max); - - let sse = |mu: f64, sigma: f64| -> f64 { - usable - .iter() - .map(|p| { - let model = standard_normal_cdf((p.a.ln() - mu) / sigma); - let d = p.events_per_half_cycle.min(1.0) - model; - d * d - }) - .sum() - }; - - let mut best = (f64::INFINITY, ln_min, 0.1); - let mu_steps = 200; - for i in 0..=mu_steps { - let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; - for j in 0..40 { - let sigma = 0.005 * 1.2_f64.powi(j); // 0.005 .. ~7 in ln a - let value = sse(mu, sigma); - if value < best.0 { - best = (value, mu, sigma); - } - } - } - let (sse_min, mu_hat, sigma_hat) = best; - let dof = usable.len().saturating_sub(2).max(1) as f64; - let threshold = sse_min * (1.0 + 2.0 / dof) + 1e-12; - - // Profile over mu: the interval where some sigma keeps SSE under the - // threshold. - let mut low = mu_hat; - let mut high = mu_hat; - for i in 0..=mu_steps { - let mu = ln_min + (ln_max - ln_min) * i as f64 / mu_steps as f64; - let feasible = (0..40).any(|j| { - let sigma = 0.005 * 1.2_f64.powi(j); - sse(mu, sigma) <= threshold - }); - if feasible { - low = low.min(mu); - high = high.max(mu); - } - } - - Some(MinDepthFit { - a_min: mu_hat.exp(), - a_min_low: low.exp(), - a_min_high: high.exp(), - sigma_ln_a: sigma_hat, - points_used: usable.len(), - }) -} - -// --------------------------------------------------------------------------- -// Hot-pixel mask (background is heavy-tailed; mask the tail, use the body) -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone)] -pub struct HotPixelMask { - width: u16, - masked: Vec, -} - -impl HotPixelMask { - /// Builds the mask from per-pixel counts of an *unmodulated* reference - /// window: pixels above `median + 5·MAD` (and above a small absolute - /// floor) are masked. The mask is fixed-pattern and belongs in the run - /// metadata, not just preprocessing. - pub fn from_reference_counts(width: u16, _height: u16, counts: &[u32]) -> Self { - let mut sorted: Vec = counts.to_vec(); - sorted.sort_unstable(); - let median = sorted.get(sorted.len() / 2).copied().unwrap_or(0) as f64; - let mut deviations: Vec = counts - .iter() - .map(|&count| (f64::from(count) - median).abs()) - .collect(); - deviations.sort_by(f64::total_cmp); - let mad = deviations.get(deviations.len() / 2).copied().unwrap_or(0.0); - let threshold = median + 5.0 * mad.max(0.5) + 2.0; - let masked = counts - .iter() - .map(|&count| f64::from(count) > threshold) - .collect(); - Self { width, masked } - } - - pub fn is_masked(&self, x: u16, y: u16) -> bool { - self.masked - .get(y as usize * self.width as usize + x as usize) - .copied() - .unwrap_or(false) - } - - pub fn masked_count(&self) -> usize { - self.masked.iter().filter(|&&m| m).count() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Deterministic pseudo-uniform stream (splitmix64 → [0,1)). - struct UniformStream { - state: u64, - } - - impl UniformStream { - fn new(seed: u64) -> Self { - Self { state: seed } - } - - fn next(&mut self) -> f64 { - self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.state; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z = z ^ (z >> 31); - (z >> 11) as f64 / (1_u64 << 53) as f64 - } - - fn take(&mut self, n: usize) -> Vec { - (0..n).map(|_| self.next()).collect() - } - } - - fn uniform_sequence(seed: u64, n: usize) -> Vec { - UniformStream::new(seed).take(n) - } - - /// Synthetic event stream: `per_cycle` phase-locked events per cycle at - /// `locked_phase` (jitter ±0.02) plus `background_rate_hz` uniform noise. - fn synthetic_events( - frequency_hz: f64, - duration_s: f64, - per_cycle: f64, - background_rate_hz: f64, - seed: u64, - ) -> Vec { - let cycles = (frequency_hz * duration_s) as usize; - let period_us = 1.0e6 / frequency_hz; - let mut stream = UniformStream::new(seed); - let mut next = move || stream.next(); - let mut events = Vec::new(); - for cycle in 0..cycles { - let base = cycle as f64 * period_us; - // Bernoulli(per_cycle fractional part) + floor. - let mut count = per_cycle.floor() as usize; - if next() < per_cycle.fract() { - count += 1; - } - for _ in 0..count { - let phase = 0.25 + (next() - 0.5) * 0.04; - events.push((base + phase * period_us) as u64); - } - } - let n_background = (background_rate_hz * duration_s) as usize; - for _ in 0..n_background { - events.push((next() * duration_s * 1.0e6) as u64); - } - events.sort_unstable(); - events - } - - #[test] - fn rayleigh_accepts_uniform_and_rejects_locked_phases() { - let uniform = uniform_sequence(7, 2_000); - let stat = rayleigh_test(&uniform); - assert!(stat.p_value > 0.01, "uniform phases: p={}", stat.p_value); - - let locked: Vec = uniform_sequence(11, 200) - .into_iter() - .map(|u| 0.3 + 0.02 * (u - 0.5)) - .collect(); - let stat = rayleigh_test(&locked); - assert!(stat.p_value < 1e-12, "locked phases: p={}", stat.p_value); - } - - #[test] - fn detection_discounts_uniform_background() { - // 0.8 locked events/cycle at 1 kHz for 0.5 s, drowned in 10x - // background rate: still detected via phase. - let events = synthetic_events(1_000.0, 0.5, 0.8, 8_000.0, 3); - let phases = fold_phases(events.iter().copied(), 0, 1_000.0); - let stat = rayleigh_test(&phases); - assert!(stat.p_value < 1e-6, "p={}", stat.p_value); - - // Background alone must NOT detect. - let noise_only = synthetic_events(1_000.0, 0.5, 0.0, 8_000.0, 5); - let phases = fold_phases(noise_only.iter().copied(), 0, 1_000.0); - let stat = rayleigh_test(&phases); - assert!(stat.p_value > 1e-3, "background-only p={}", stat.p_value); - } - - #[test] - fn phase_locked_excess_recovers_events_per_cycle() { - let frequency = 2_000.0; - let duration = 0.5; - let per_cycle = 0.6; - let events = synthetic_events(frequency, duration, per_cycle, 2_000.0, 9); - let phases = fold_phases(events.iter().copied(), 0, frequency); - let histogram = phase_histogram(&phases, 32); - let cycles = frequency * duration; - let recovered = phase_locked_excess(&histogram) / cycles; - assert!( - (recovered - per_cycle).abs() < 0.12, - "recovered {recovered} vs {per_cycle}" - ); - } - - #[test] - fn frequency_refinement_recovers_clock_skew() { - // Commanded 5 kHz, true (camera-clock) frequency 300 ppm higher — - // the naive fold dephases by 1.5 cycles over the 1 s span and - // collapses, while the refined lock recovers the true frequency. - let true_hz = 5_000.0 * (1.0 + 300e-6); - let events = synthetic_events(true_hz, 1.0, 1.0, 500.0, 13); - let lock = refine_frequency(&events, 5_000.0, 500.0).expect("lock found"); - let recovered_ppm = (lock.frequency_hz / 5_000.0 - 1.0) * 1e6; - // The scan step is 0.1/span = 0.1 Hz = 20 ppm at 5 kHz. - assert!( - (recovered_ppm - 300.0).abs() < 25.0, - "recovered {recovered_ppm} ppm" - ); - let naive = rayleigh_test(&fold_phases(events.iter().copied(), events[0], 5_000.0)); - assert!( - lock.rayleigh.z > naive.z * 5.0, - "lock z={} naive z={}", - lock.rayleigh.z, - naive.z - ); - } - - #[test] - fn fiducial_folding_matches_known_phase() { - let cycle_starts: Vec = (0..100).map(|k| k * 1_000).collect(); - let events: Vec = (0..99).map(|k| k * 1_000 + 250).collect(); - let phases = fold_phases_with_fiducials(&events, &cycle_starts); - assert_eq!(phases.len(), 99); - assert!(phases.iter().all(|p| (p - 0.25).abs() < 1e-9)); - } - - #[test] - fn min_depth_fit_recovers_the_crossing() { - // True a_min = 0.20, smear sigma = 0.15 in ln a. - let mu = 0.2_f64.ln(); - let points: Vec = (0..12) - .map(|i| { - let a = 0.08 * 1.25_f64.powi(i); // 0.08 .. ~0.9 - DepthPoint { - a, - events_per_half_cycle: standard_normal_cdf((a.ln() - mu) / 0.15), - } - }) - .collect(); - let fit = fit_min_depth(&points).expect("fit succeeds"); - assert!( - (fit.a_min - 0.2).abs() < 0.02, - "a_min={} (expected 0.20)", - fit.a_min - ); - assert!(fit.a_min_low <= fit.a_min && fit.a_min <= fit.a_min_high); - assert!((fit.sigma_ln_a - 0.15).abs() < 0.08); - } - - #[test] - fn min_depth_fit_requires_a_bracketed_transition() { - // All points fully above threshold: no crossing to fit. - let points: Vec = (0..6) - .map(|i| DepthPoint { - a: 0.5 + 0.1 * i as f64, - events_per_half_cycle: 1.0, - }) - .collect(); - assert!(fit_min_depth(&points).is_none()); - } - - #[test] - fn hot_pixel_mask_flags_the_tail_only() { - let mut counts = vec![2_u32; 64 * 64]; - counts[5] = 500; // hot - counts[700] = 300; // hot - let mask = HotPixelMask::from_reference_counts(64, 64, &counts); - assert_eq!(mask.masked_count(), 2); - assert!(mask.is_masked(5, 0)); - assert!(!mask.is_masked(6, 0)); - } -} diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs deleted file mode 100644 index 808be04..0000000 --- a/plugins/stage-a-a1/src/lib.rs +++ /dev/null @@ -1,1254 +0,0 @@ -//! Stage-A A1 — event-native Bode calibration, minimum-depth method. -//! -//! Measures `a_min(f)`: the smallest optical log-contrast that still -//! produces phase-locked events, per drive frequency. `|H(f)| = -//! C/a_min(f)`, the knee is `f_c(I)`, and the plateau of `a_min` reads out -//! the contrast quantum `C` (knowledge base: -//! `methodology/camera-calibration.md`, A1 protocol). -//! -//! Division of labour: -//! - `analysis` — phase folding, Rayleigh detection, frequency-skew -//! recovery, phase-locked excess, probit `a_min` fit, hot-pixel mask; -//! - `sweep` — the per-frequency bisection/grid state machine; -//! - this module — device I/O through `stage-a-io` (gated by the ABI v5 -//! execution context), camera-event intake, measurement windows, live -//! views, and the run sidecar. -//! -//! ON and OFF are measured **separately** (never pooled — the paths are -//! asymmetric); select the polarity in the settings and run each sweep. - -mod analysis; -mod sweep; - -use std::collections::BTreeMap; -use std::path::PathBuf; - -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, PluginInput, - Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, - TableSchema, TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, -}; -use serde_json::{json, Value}; -use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, DeviceEvent, FrameType, IoWorker, PdqWriter, - RunSidecar, StageAClient, StreamIntegrity, TriggerSource, WorkerOutput, WorkerRequest, -}; - -use analysis::{ - detect, fold_phases, fold_phases_with_fiducials, phase_histogram, phase_locked_excess, - rayleigh_test, refine_frequency, HotPixelMask, PhaseHistogram, -}; -use sweep::{Measurement, SweepCommand, SweepEngine, SweepPlan}; - -const AMIN_DATASET_ID: &str = "stage-a-a1.amin"; -const PHASE_DATASET_ID: &str = "stage-a-a1.phase"; -const DEPTH_DATASET_ID: &str = "stage-a-a1.depth"; -const STATUS_DATASET_ID: &str = "stage-a-a1.status"; - -const ACTION_ARM: &str = "stage-a-a1.arm"; -const ACTION_RUN: &str = "stage-a-a1.run"; -const ACTION_STOP: &str = "stage-a-a1.stop"; - -const PHASE_BINS: usize = 32; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RunState { - Idle, - Armed, - Reference, - Sweeping, - Finished, -} - -/// Camera-side accumulation for the current measurement window. -#[derive(Default)] -struct WindowAccumulator { - /// Camera timestamps of polarity-selected, hot-pixel-filtered events. - event_timestamps_us: Vec, - /// Rising phase-0 trigger edges (cycle fiducials) inside the window. - trigger_edges_us: Vec, - /// ADC codes streamed by the Teensy during the window. - adc_codes: Vec, - window_start_us: Option, - latest_camera_ts_us: u64, -} - -impl WindowAccumulator { - fn clear(&mut self) { - self.event_timestamps_us.clear(); - self.trigger_edges_us.clear(); - self.adc_codes.clear(); - self.window_start_us = None; - } - - fn elapsed_us(&self) -> u64 { - self.window_start_us - .map(|start| self.latest_camera_ts_us.saturating_sub(start)) - .unwrap_or(0) - } -} - -pub struct StageAA1Plugin { - enabled: bool, - state: RunState, - // device - worker: Option, - next_tag: u64, - in_flight: BTreeMap, - firmware: String, - integrity: StreamIntegrity, - last_error: Option, - effects_blocked_reason: Option, - // configuration (settings) - port_hint: String, - polarity_on: bool, - freq_start_hz: f64, - freq_stop_hz: f64, - points_per_decade: i64, - cycles_per_measurement: i64, - settle_ms: i64, - alpha: f64, - initial_amplitude_dac: i64, - sample_rate_hz: i64, - calibration: AdcCalibration, - // run - engine: Option, - window: WindowAccumulator, - settle_until_us: Option, - hot_pixels: Option, - reference_counts: Vec, - sensor_size: (u16, u16), - run_id: String, - /// CONFIG reply fields exactly as the controller ACKed them (sidecar). - last_acked_config: BTreeMap, - pdq: Option, - current_phase_histogram: Option, - used_hardware_fiducial: bool, - dataset_generation: u64, - consumed_action_ids: Vec, -} - -impl Default for StageAA1Plugin { - fn default() -> Self { - Self { - enabled: false, - state: RunState::Idle, - worker: None, - next_tag: 1, - in_flight: BTreeMap::new(), - firmware: String::new(), - integrity: StreamIntegrity::default(), - last_error: None, - effects_blocked_reason: None, - port_hint: "auto".into(), - polarity_on: true, - freq_start_hz: 100.0, - freq_stop_hz: 50_000.0, - points_per_decade: 6, - cycles_per_measurement: 400, - settle_ms: 100, - alpha: 0.001, - initial_amplitude_dac: 512, - sample_rate_hz: 20_000, - calibration: AdcCalibration::default(), - engine: None, - window: WindowAccumulator::default(), - settle_until_us: None, - hot_pixels: None, - reference_counts: Vec::new(), - sensor_size: (0, 0), - run_id: String::new(), - last_acked_config: BTreeMap::new(), - pdq: None, - current_phase_histogram: None, - used_hardware_fiducial: false, - dataset_generation: 0, - consumed_action_ids: Vec::new(), - } - } -} - -impl StageAA1Plugin { - fn bump(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn frequency_grid(&self) -> Vec { - let start = self.freq_start_hz.max(1.0); - let stop = self.freq_stop_hz.max(start * 1.01); - let per_decade = self.points_per_decade.max(1) as f64; - let decades = (stop / start).log10(); - let n = (decades * per_decade).ceil() as usize + 1; - (0..n) - .map(|i| start * 10f64.powf(i as f64 / per_decade)) - .filter(|&f| f <= stop * 1.0001) - .collect() - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - - fn arm(&mut self) { - if self.worker.is_some() { - return; - } - match open_transport(&self.port_hint) { - Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - self.state = RunState::Armed; - self.last_error = None; - } - Err(err) => self.last_error = Some(err), - } - self.bump(); - } - - fn start_run(&mut self) { - if self.worker.is_none() { - self.last_error = Some("run: arm the controller first".into()); - return; - } - self.run_id = format!( - "A1-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) - ); - let pdq_path = run_data_dir().join(format!("{}.pdq", self.run_id)); - match PdqWriter::create(&pdq_path) { - Ok(writer) => self.pdq = Some(writer), - Err(err) => { - self.last_error = Some(format!("pdq: {err}")); - return; - } - } - let plan = SweepPlan { - frequencies_hz: self.frequency_grid(), - initial_amplitude_dac: self.initial_amplitude_dac.clamp(1, 2_047) as u32, - ..SweepPlan::default() - }; - self.engine = Some(SweepEngine::new(plan)); - self.reference_counts.clear(); - self.hot_pixels = None; - self.window.clear(); - self.settle_until_us = None; - // Reference phase: unmodulated field (amplitude 0) for the - // hot-pixel mask and the background sanity check. - self.send_drive(self.freq_start_hz, 0, "reference"); - self.state = RunState::Reference; - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(true)); - } - self.bump(); - } - - fn stop_run(&mut self, reason: &str) { - self.queue_command("stop", Command::new("STOP").field("reason", reason)); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - self.finish_run(); - self.state = if self.worker.is_some() { - RunState::Armed - } else { - RunState::Idle - }; - self.bump(); - } - - fn disarm(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - worker.shutdown(reason); - } - self.finish_run(); - self.in_flight.clear(); - self.state = RunState::Idle; - self.bump(); - } - - fn finish_run(&mut self) { - if let Some(pdq) = self.pdq.take() { - match pdq.finish(self.integrity) { - Ok(summary) => { - let mut sidecar = RunSidecar::from_pdq(&self.run_id, "A1", &summary); - sidecar.plugin_name = "stage-a-a1".into(); - sidecar.plugin_version = env!("CARGO_PKG_VERSION").into(); - sidecar.firmware_version = self.firmware.clone(); - sidecar.adc_calibration = self.calibration.clone(); - sidecar.configured_sample_rate_hz = self.sample_rate_hz as u32; - sidecar.acked_config = self.last_acked_config.clone(); - sidecar.trigger_source = if self.used_hardware_fiducial { - TriggerSource::DrivePhase0 - } else { - TriggerSource::None - }; - sidecar.valid = summary.valid; - if let Some(mask) = &self.hot_pixels { - sidecar - .notes - .push(format!("hot pixels masked: {}", mask.masked_count())); - } - let sidecar_path = run_data_dir().join(format!("{}.json", self.run_id)); - if let Err(err) = sidecar.write_json(&sidecar_path) { - self.last_error = Some(format!("sidecar: {err}")); - } - if let Some(engine) = &self.engine { - let results_path = - run_data_dir().join(format!("{}.results.json", self.run_id)); - let _ = std::fs::write( - &results_path, - serde_json::to_vec_pretty(&results_json(engine)).unwrap_or_default(), - ); - } - } - Err(err) => self.last_error = Some(format!("pdq finish: {err}")), - } - } - } - - fn send_drive(&mut self, frequency_hz: f64, amplitude_dac: u32, purpose: &str) { - let freq_mhz = (frequency_hz * 1_000.0).round() as i64; - // The firmware only accepts CONFIG from SAFE_IDLE/CONFIGURED, so - // every new drive point must stop the running acquisition first - // (STOP is idempotent and harmless before the first point). - self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); - self.queue_command( - purpose, - Command::new("CONFIG") - .field("mode", "A1") - .field("wave", "SINE") - .field("freq_mhz", freq_mhz) - .field("center_dac", 2_048) - .field("amplitude_dac", amplitude_dac) - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", 256) - .field("raw", 1) - .field("summary", 1), - ); - self.queue_command("start", Command::new("START")); - self.window.clear(); - self.settle_until_us = None; // set on the first camera frame seen - self.current_phase_histogram = None; - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - let mut stopped = None; - let mut watchdog_fault: Option = None; - for output in outputs { - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => match purpose.as_str() { - "hello" => { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - } - // CONFIG ACKs (drive points) go into the sidecar - // verbatim, per the control-software spec. - "reference" | "sweep" => { - self.last_acked_config = fields; - } - _ => {} - }, - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - WorkerOutput::Event(DeviceEvent::Data(frame)) => { - if let Some(pdq) = &mut self.pdq { - let _ = pdq.write_frame(&frame); - } - if frame.header.frame_type == FrameType::SamplesU16 { - if let Some(codes) = frame.samples() { - self.window.adc_codes.extend_from_slice(&codes); - } - } - } - WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { - if name == "FAULT" { - watchdog_fault = Some( - fields - .get("code") - .cloned() - .unwrap_or_else(|| "unknown".into()), - ); - } - } - WorkerOutput::Integrity(integrity) => self.integrity = integrity, - WorkerOutput::Stopped { reason } => stopped = Some(reason), - } - } - if let Some(code) = watchdog_fault { - // The controller safed itself mid-run; the current point is - // invalid and the run cannot silently continue. - self.last_error = Some(format!("controller fault: {code} — run aborted")); - if matches!(self.state, RunState::Reference | RunState::Sweeping) { - self.stop_run("watchdog_fault"); - } - } - if let Some(reason) = stopped { - self.worker = None; - self.last_error = Some(format!("device connection ended: {reason}")); - self.finish_run(); - self.state = RunState::Idle; - self.bump(); - } - } - - fn ingest_camera_frame(&mut self, frame: &PluginFrame<'_>) { - self.sensor_size = (frame.width(), frame.height()); - self.window.latest_camera_ts_us = frame.window_end_us(); - if self.settle_until_us.is_none() { - self.settle_until_us = - Some(frame.window_end_us() + (self.settle_ms.max(0) as u64) * 1_000); - return; - } - let settle_until = self.settle_until_us.unwrap_or(0); - if frame.window_end_us() < settle_until { - return; - } - self.window - .window_start_us - .get_or_insert(frame.window_start_us()); - - if self.state == RunState::Reference { - if self.reference_counts.len() != frame.width() as usize * frame.height() as usize { - self.reference_counts = vec![0; frame.width() as usize * frame.height() as usize]; - } - for event in frame.events() { - let idx = event.y as usize * frame.width() as usize + event.x as usize; - if let Some(slot) = self.reference_counts.get_mut(idx) { - *slot += 1; - } - } - } else { - let mask = self.hot_pixels.as_ref(); - for event in frame.events() { - if event.is_on() != self.polarity_on { - continue; - } - if mask.is_some_and(|m| m.is_masked(event.x, event.y)) { - continue; - } - self.window.event_timestamps_us.push(event.timestamp_us()); - } - } - for trigger in frame.external_triggers() { - if trigger.is_rising() { - self.window.trigger_edges_us.push(trigger.timestamp_us); - } - } - } - - fn window_target_us(&self, frequency_hz: f64) -> u64 { - ((self.cycles_per_measurement.max(10) as f64 / frequency_hz) * 1.0e6) as u64 - } - - fn advance_run(&mut self) { - match self.state { - RunState::Reference => { - // A fixed 0.5 s of unmodulated reference. - if self.window.elapsed_us() < 500_000 { - return; - } - let (width, height) = self.sensor_size; - if width > 0 && !self.reference_counts.is_empty() { - self.hot_pixels = Some(HotPixelMask::from_reference_counts( - width, - height, - &self.reference_counts, - )); - } - self.state = RunState::Sweeping; - let Some(engine) = &self.engine else { - return; - }; - if let SweepCommand::Measure { - frequency_hz, - amplitude_dac, - } = engine.current_command() - { - self.send_drive(frequency_hz, amplitude_dac, "sweep"); - } - self.bump(); - } - RunState::Sweeping => { - let Some(engine) = &self.engine else { - return; - }; - let SweepCommand::Measure { - frequency_hz, - amplitude_dac, - } = engine.current_command() - else { - self.state = RunState::Finished; - self.finish_run(); - self.bump(); - return; - }; - if self.window.elapsed_us() < self.window_target_us(frequency_hz) { - return; - } - let measurement = self.evaluate_window(frequency_hz, amplitude_dac); - let next = { - let engine = self.engine.as_mut().expect("engine exists"); - engine.ingest(measurement) - }; - match next { - SweepCommand::Measure { - frequency_hz, - amplitude_dac, - } => self.send_drive(frequency_hz, amplitude_dac, "sweep"), - SweepCommand::Finished => { - self.queue_command("stop", Command::new("STOP").field("reason", "done")); - self.state = RunState::Finished; - self.finish_run(); - } - } - self.bump(); - } - _ => {} - } - } - - fn evaluate_window(&mut self, frequency_hz: f64, amplitude_dac: u32) -> Measurement { - // Optical contrast from the photodiode trace; any estimator - // rejection or stream fault invalidates the point. - let measured_a = if self.integrity.is_clean() { - estimate_contrast(&self.window.adc_codes, &self.calibration) - .ok() - .map(|estimate| estimate.a) - } else { - None - }; - - let events = &self.window.event_timestamps_us; - let observed_cycles = self.window.elapsed_us() as f64 / 1.0e6 * frequency_hz; - - // Cycle fiducial: hardware phase-0 edges when present, otherwise - // software frequency refinement against the events themselves. - let (phases, trials) = if self.window.trigger_edges_us.len() >= 2 { - self.used_hardware_fiducial = true; - ( - fold_phases_with_fiducials(events, &self.window.trigger_edges_us), - 1, - ) - } else if let Some(lock) = refine_frequency(events, frequency_hz, 100.0) { - ( - fold_phases( - events.iter().copied(), - events.first().copied().unwrap_or(0), - lock.frequency_hz, - ), - lock.trials, - ) - } else { - (Vec::new(), 1) - }; - - let stat = rayleigh_test(&phases); - let histogram = phase_histogram(&phases, PHASE_BINS); - let excess = phase_locked_excess(&histogram); - self.current_phase_histogram = Some(histogram); - let verdict = detect(stat, excess, observed_cycles, self.alpha, trials); - - Measurement { - amplitude_dac, - measured_a, - events_per_half_cycle: verdict.locked_events_per_cycle, - detected: verdict.detected, - } - } - - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) - || !request.action_id.starts_with("stage-a-a1.") - { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed - } - - // -- datasets -------------------------------------------------------- - - fn amin_dataset(&self) -> Series1dV1 { - let mut a_min = Vec::new(); - let mut low = Vec::new(); - let mut high = Vec::new(); - if let Some(engine) = &self.engine { - for result in &engine.results { - if let Some(fit) = &result.fit { - a_min.push(Series1dPoint { - x: result.frequency_hz, - y: fit.a_min, - }); - low.push(Series1dPoint { - x: result.frequency_hz, - y: fit.a_min_low, - }); - high.push(Series1dPoint { - x: result.frequency_hz, - y: fit.a_min_high, - }); - } - } - } - Series1dV1 { - x_label: "drive frequency [Hz]".into(), - y_label: "a_min".into(), - lines: vec![ - Series1dLine { - name: "a_min".into(), - points: a_min, - }, - Series1dLine { - name: "CI low".into(), - points: low, - }, - Series1dLine { - name: "CI high".into(), - points: high, - }, - ], - } - } - - fn phase_dataset(&self) -> Series1dV1 { - let points = self - .current_phase_histogram - .as_ref() - .map(|histogram| { - histogram - .bins - .iter() - .enumerate() - .map(|(i, &count)| Series1dPoint { - x: (i as f64 + 0.5) / histogram.bins.len() as f64, - y: f64::from(count), - }) - .collect() - }) - .unwrap_or_default(); - Series1dV1 { - x_label: "drive phase [cycles]".into(), - y_label: "events".into(), - lines: vec![Series1dLine { - name: if self.polarity_on { "ON" } else { "OFF" }.into(), - points, - }], - } - } - - fn depth_dataset(&self) -> Series1dV1 { - let mut points: Vec = self - .engine - .as_ref() - .map(|engine| { - let mut all: Vec = engine - .results - .last() - .map(|result| { - result - .points - .iter() - .map(|p| Series1dPoint { - x: p.a, - y: p.events_per_half_cycle, - }) - .collect() - }) - .unwrap_or_default(); - all.sort_by(|p, q| p.x.total_cmp(&q.x)); - all - }) - .unwrap_or_default(); - points.dedup_by(|p, q| p.x == q.x); - Series1dV1 { - x_label: "measured a".into(), - y_label: "locked events / half-cycle".into(), - lines: vec![Series1dLine { - name: "N(a)".into(), - points, - }], - } - } - - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("progress", "Progress"), - column("fiducial", "Cycle fiducial"), - column("hot_pixels", "Hot pixels"), - column("integrity", "Integrity"), - column("error", "Last error"), - ], - ..TableSchema::default() - } - } - - fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.state) { - (Some(reason), _) => format!("locked ({reason})"), - (None, RunState::Idle) => "idle".into(), - (None, RunState::Armed) => format!("armed ({})", self.firmware), - (None, RunState::Reference) => "reference window (hot-pixel mask)".into(), - (None, RunState::Sweeping) => "sweeping".into(), - (None, RunState::Finished) => "finished".into(), - }; - let progress = self - .engine - .as_ref() - .map(|engine| { - format!( - "{}/{} frequencies", - engine.results.len(), - engine.results.len() + if engine.is_finished() { 0 } else { 1 } - ) - }) - .unwrap_or_else(|| "—".into()); - let fiducial = if self.used_hardware_fiducial { - "EXT_TRIGGER phase-0".to_owned() - } else { - "software frequency lock".to_owned() - }; - let hot = self - .hot_pixels - .as_ref() - .map(|mask| format!("{} masked", mask.masked_count())) - .unwrap_or_else(|| "—".into()); - let integrity = if self.integrity.is_clean() { - "clean".to_owned() - } else { - format!( - "crc={} gaps={} overruns={}", - self.integrity.crc_failures, - self.integrity.sequence_gaps, - self.integrity.dropped_samples - ) - }; - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", state), - text_column("progress", progress), - text_column("fiducial", fiducial), - text_column("hot_pixels", hot), - text_column("integrity", integrity), - text_column("error", self.last_error.clone().unwrap_or_default()), - ], - } - } -} - -fn results_json(engine: &SweepEngine) -> Value { - json!({ - "results": engine - .results - .iter() - .map(|result| { - json!({ - "frequency_hz": result.frequency_hz, - "exhausted": result.exhausted, - "measurements": result.measurements, - "fit": result.fit.as_ref().map(|fit| json!({ - "a_min": fit.a_min, - "a_min_low": fit.a_min_low, - "a_min_high": fit.a_min_high, - "sigma_ln_a": fit.sigma_ln_a, - "points_used": fit.points_used, - })), - "points": result - .points - .iter() - .map(|p| json!({"a": p.a, "events_per_half_cycle": p.events_per_half_cycle})) - .collect::>(), - }) - }) - .collect::>(), - }) -} - -fn run_data_dir() -> PathBuf { - let home = std::env::var_os("HOME") - .map(PathBuf::from) - .unwrap_or_default(); - home.join(".augur").join("stage-a-runs") -} - -fn open_transport(port_hint: &str) -> Result, String> { - let path = if port_hint == "auto" { - stage_a_io::transport::available_port_names() - .into_iter() - .find(|name| name.contains("usbmodem") || name.contains("ttyACM")) - .ok_or_else(|| "no USB serial device found".to_owned())? - } else { - port_hint.to_owned() - }; - let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} - -impl Plugin for StageAA1Plugin { - fn name(&self) -> &'static str { - "Stage-A A1 Min-Depth" - } - - fn description(&self) -> &'static str { - "Event-native Bode calibration: a_min(f) via phase-locked detection, bisection, and probit fitting." - } - - fn enabled(&self) -> bool { - self.enabled - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.disarm("plugin disabled"); - } - } - - fn reset(&mut self) { - self.window.clear(); - self.current_phase_histogram = None; - self.bump(); - } - - fn input_kind(&self) -> PluginInput { - PluginInput::RawEvents - } - - fn process_frame( - &mut self, - frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = - Some(format!("effects not allowed in {:?}", execution.mode)); - if self.worker.is_some() { - self.disarm("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for action in self.consume_actions(context) { - match action.as_str() { - ACTION_ARM => self.arm(), - ACTION_RUN => self.start_run(), - ACTION_STOP => self.stop_run("operator"), - _ => {} - } - } - - self.drain_worker(); - if matches!(self.state, RunState::Reference | RunState::Sweeping) { - self.ingest_camera_frame(frame); - self.advance_run(); - } - } - - fn settings_schema(&self) -> SettingsSchema { - SettingsSchema { - sections: vec![ - SettingsSection { - label: "Sweep".into(), - description: Some( - "Frequency grid and statistics. ON and OFF are measured in separate \ - runs — never pooled." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "freq_start_hz".into(), - label: "Start frequency".into(), - tooltip: None, - kind: SettingKind::F64Drag { - min: 1.0, - max: 1.0e6, - speed: 10.0, - default: self.freq_start_hz, - }, - }, - SettingItem { - key: "freq_stop_hz".into(), - label: "Stop frequency".into(), - tooltip: None, - kind: SettingKind::F64Drag { - min: 1.0, - max: 1.0e6, - speed: 100.0, - default: self.freq_stop_hz, - }, - }, - SettingItem { - key: "points_per_decade".into(), - label: "Points per decade".into(), - tooltip: None, - kind: SettingKind::I64Slider { - min: 2, - max: 12, - default: self.points_per_decade, - suffix: None, - }, - }, - SettingItem { - key: "cycles_per_measurement".into(), - label: "Cycles per measurement".into(), - tooltip: Some( - "Modulation cycles integrated per amplitude point".into(), - ), - kind: SettingKind::I64Slider { - min: 50, - max: 5_000, - default: self.cycles_per_measurement, - suffix: None, - }, - }, - SettingItem { - key: "polarity_on".into(), - label: "Polarity".into(), - tooltip: Some("Which comparator path this sweep measures".into()), - kind: SettingKind::Enum { - variants: vec!["ON".into(), "OFF".into()], - default: usize::from(!self.polarity_on), - }, - }, - SettingItem { - key: "alpha".into(), - label: "Significance α".into(), - tooltip: Some( - "Per-measurement false-positive budget (Bonferroni-corrected \ - for the frequency scan)" - .into(), - ), - kind: SettingKind::F64Drag { - min: 1e-6, - max: 0.05, - speed: 1e-4, - default: self.alpha, - }, - }, - ], - }, - SettingsSection { - label: "Device".into(), - description: None, - default_open: false, - items: vec![ - SettingItem { - key: "initial_amplitude_dac".into(), - label: "Initial amplitude (DAC)".into(), - tooltip: None, - kind: SettingKind::I64Slider { - min: 1, - max: 2_047, - default: self.initial_amplitude_dac, - suffix: None, - }, - }, - SettingItem { - key: "settle_ms".into(), - label: "Settle time".into(), - tooltip: Some( - "Discarded after each drive change (HVA/Pockels settling + \ - refractory clearing)" - .into(), - ), - kind: SettingKind::I64Slider { - min: 10, - max: 2_000, - default: self.settle_ms, - suffix: Some(" ms".into()), - }, - }, - SettingItem { - key: "dark_millivolts".into(), - label: "Dark level".into(), - tooltip: None, - kind: SettingKind::F64Drag { - min: 0.0, - max: 3_300.0, - speed: 1.0, - default: self.calibration.dark_volts * 1_000.0, - }, - }, - ], - }, - ], - } - } - - fn get_setting(&self, key: &str) -> Option { - match key { - "freq_start_hz" => Some(json!(self.freq_start_hz)), - "freq_stop_hz" => Some(json!(self.freq_stop_hz)), - "points_per_decade" => Some(json!(self.points_per_decade)), - "cycles_per_measurement" => Some(json!(self.cycles_per_measurement)), - "polarity_on" => Some(json!(if self.polarity_on { "ON" } else { "OFF" })), - "alpha" => Some(json!(self.alpha)), - "initial_amplitude_dac" => Some(json!(self.initial_amplitude_dac)), - "settle_ms" => Some(json!(self.settle_ms)), - "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "freq_start_hz" => { - self.freq_start_hz = value.as_f64().ok_or("must be a number")?.max(1.0); - } - "freq_stop_hz" => { - self.freq_stop_hz = value.as_f64().ok_or("must be a number")?.max(1.0); - } - "points_per_decade" => { - self.points_per_decade = value.as_i64().ok_or("must be an integer")?.clamp(2, 12); - } - "cycles_per_measurement" => { - self.cycles_per_measurement = - value.as_i64().ok_or("must be an integer")?.clamp(50, 5_000); - } - "polarity_on" => { - let text = value.as_str().ok_or("must be a string")?; - self.polarity_on = text.eq_ignore_ascii_case("on"); - } - "alpha" => { - self.alpha = value.as_f64().ok_or("must be a number")?.clamp(1e-6, 0.05); - } - "initial_amplitude_dac" => { - self.initial_amplitude_dac = - value.as_i64().ok_or("must be an integer")?.clamp(1, 2_047); - } - "settle_ms" => { - self.settle_ms = value.as_i64().ok_or("must be an integer")?.clamp(10, 2_000); - } - "dark_millivolts" => { - let mv = value.as_f64().ok_or("must be a number")?; - self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); - } - _ => return Err(format!("unknown setting: {key}")), - } - Ok(()) - } - - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - if let Some(engine) = &self.engine { - entries.push(StatusEntry::Text(format!( - "{} frequency points finished", - engine.results.len() - ))); - } - if let Some(err) = &self.last_error { - entries.push(StatusEntry::Text(format!("Error: {err}"))); - } - entries - } - - fn host_views(&self) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![ - HostDatasetDescriptor { - id: AMIN_DATASET_ID.into(), - title: "a_min(f)".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No fitted frequency points yet.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: PHASE_DATASET_ID.into(), - title: "Phase histogram".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No measurement window yet.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: DEPTH_DATASET_ID.into(), - title: "N(a) at current frequency".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No depth points yet.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "A1 run status".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Idle.".into(), - display: None, - relations: Vec::new(), - }, - ], - views: vec![ - HostViewDescriptor { - id: format!("{AMIN_DATASET_ID}.view"), - title: "A1 Bode (a_min)".into(), - dataset_id: AMIN_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: format!("{PHASE_DATASET_ID}.view"), - title: "Phase fold".into(), - dataset_id: PHASE_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: format!("{DEPTH_DATASET_ID}.view"), - title: "Depth staircase".into(), - dataset_id: DEPTH_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: format!("{STATUS_DATASET_ID}.view"), - title: "A1 status".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - ], - actions: vec![ - HostActionDescriptor { - id: ACTION_ARM.into(), - title: "Arm controller".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_RUN.into(), - title: "Run A1 sweep".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_STOP.into(), - title: "Stop".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - ], - } - } - - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - AMIN_DATASET_ID => serde_json::to_vec(&self.amin_dataset()).ok(), - PHASE_DATASET_ID => serde_json::to_vec(&self.phase_dataset()).ok(), - DEPTH_DATASET_ID => serde_json::to_vec(&self.depth_dataset()).ok(), - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, - } - } - - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - AMIN_DATASET_ID | PHASE_DATASET_ID | DEPTH_DATASET_ID | STATUS_DATASET_ID => { - self.dataset_generation.max(1) - } - _ => 0, - } - } -} - -impl Drop for StageAA1Plugin { - fn drop(&mut self) { - self.disarm("plugin destroyed"); - } -} - -export_plugin!(StageAA1Plugin); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn frequency_grid_is_log_spaced_and_bounded() { - let mut plugin = StageAA1Plugin::default(); - plugin.freq_start_hz = 100.0; - plugin.freq_stop_hz = 10_000.0; - plugin.points_per_decade = 4; - let grid = plugin.frequency_grid(); - assert!((grid.first().copied().unwrap() - 100.0).abs() < 1e-9); - assert!(grid.last().copied().unwrap() <= 10_000.0 * 1.001); - assert_eq!(grid.len(), 9); - for pair in grid.windows(2) { - let ratio = pair[1] / pair[0]; - assert!((ratio - 10f64.powf(0.25)).abs() < 1e-9); - } - } - - #[test] - fn status_dataset_matches_schema() { - let plugin = StageAA1Plugin::default(); - let dataset = plugin.status_dataset(); - let schema = plugin.status_schema(); - assert_eq!(dataset.columns.len(), schema.columns.len()); - } -} diff --git a/plugins/stage-a-a1/src/sweep.rs b/plugins/stage-a-a1/src/sweep.rs deleted file mode 100644 index 36f0a25..0000000 --- a/plugins/stage-a-a1/src/sweep.rs +++ /dev/null @@ -1,368 +0,0 @@ -//! Minimum-depth sweep state machine. -//! -//! For each frequency point: bisect on the integer DAC drive code until the -//! detection boundary is bracketed, then measure a small log-spaced grid -//! across the transition, then fit `a_min` (see `analysis::fit_min_depth`). -//! The engine is pure — device I/O and event analysis happen outside; it -//! only ingests finished measurements and emits the next drive request. -//! Note the asymmetry the whole design hinges on: the *search* variable is -//! the drive code, but every recorded point carries the **measured** -//! optical contrast `a` from the photodiode. - -use crate::analysis::{fit_min_depth, DepthPoint, MinDepthFit}; - -#[derive(Debug, Clone, PartialEq)] -pub struct SweepPlan { - pub frequencies_hz: Vec, - pub initial_amplitude_dac: u32, - pub max_amplitude_dac: u32, - /// Grid points measured across the bracket after bisection. - pub grid_points: usize, - /// Hard cap on measurements per frequency (bisection + grid). - pub max_measurements_per_frequency: usize, -} - -impl Default for SweepPlan { - fn default() -> Self { - Self { - frequencies_hz: Vec::new(), - initial_amplitude_dac: 512, - max_amplitude_dac: 2_047, - grid_points: 6, - max_measurements_per_frequency: 24, - } - } -} - -/// One finished measurement at the currently requested drive. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct Measurement { - pub amplitude_dac: u32, - /// Photodiode-measured optical log-contrast. `None` = invalid window - /// (clipped / integrity fault) — the point is discarded and re-measured. - pub measured_a: Option, - pub events_per_half_cycle: f64, - pub detected: bool, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum SweepCommand { - /// Configure the drive and measure at these settings. - Measure { - frequency_hz: f64, - amplitude_dac: u32, - }, - /// All frequencies finished. - Finished, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct FrequencyResult { - pub frequency_hz: f64, - pub fit: Option, - pub points: Vec, - pub measurements: usize, - /// True when the point budget ran out before the transition was - /// bracketed — a_min is not identifiable from this data. - pub exhausted: bool, -} - -#[derive(Debug, Clone, PartialEq)] -enum Phase { - Bisecting, - Grid { queue: Vec }, -} - -pub struct SweepEngine { - plan: SweepPlan, - frequency_index: usize, - phase: Phase, - current_dac: u32, - measurements_at_frequency: usize, - /// Highest drive code that did NOT detect / lowest that did. - highest_undetected: Option, - lowest_detected: Option, - points: Vec, - invalid_retries: usize, - pub results: Vec, -} - -impl SweepEngine { - pub fn new(plan: SweepPlan) -> Self { - let current_dac = plan.initial_amplitude_dac; - Self { - plan, - frequency_index: 0, - phase: Phase::Bisecting, - current_dac, - measurements_at_frequency: 0, - highest_undetected: None, - lowest_detected: None, - points: Vec::new(), - invalid_retries: 0, - results: Vec::new(), - } - } - - pub fn current_command(&self) -> SweepCommand { - match self.plan.frequencies_hz.get(self.frequency_index) { - Some(&frequency_hz) => SweepCommand::Measure { - frequency_hz, - amplitude_dac: self.current_dac, - }, - None => SweepCommand::Finished, - } - } - - pub fn is_finished(&self) -> bool { - self.frequency_index >= self.plan.frequencies_hz.len() - } - - /// Ingests the finished measurement for the last `Measure` command and - /// advances the state machine. - pub fn ingest(&mut self, measurement: Measurement) -> SweepCommand { - if self.is_finished() { - return SweepCommand::Finished; - } - - let Some(a) = measurement.measured_a else { - // Invalid window: re-measure the same point (bounded retries), - // never silently keep the previous contrast. - self.invalid_retries += 1; - if self.invalid_retries > 3 { - self.finish_frequency(true); - } - return self.current_command(); - }; - self.invalid_retries = 0; - self.measurements_at_frequency += 1; - self.points.push(DepthPoint { - a, - events_per_half_cycle: measurement.events_per_half_cycle, - }); - - if measurement.detected { - self.lowest_detected = - Some(self.lowest_detected.map_or(measurement.amplitude_dac, |d| { - d.min(measurement.amplitude_dac) - })); - } else { - self.highest_undetected = Some( - self.highest_undetected - .map_or(measurement.amplitude_dac, |d| { - d.max(measurement.amplitude_dac) - }), - ); - } - - if self.measurements_at_frequency >= self.plan.max_measurements_per_frequency { - self.finish_frequency(!self.bracketed()); - return self.current_command(); - } - - match &mut self.phase { - Phase::Bisecting => { - if self.bracketed() { - let queue = self.grid_queue(); - self.phase = Phase::Grid { queue }; - self.advance_grid(); - } else if measurement.detected { - // Drive down toward the boundary. - let next = ((measurement.amplitude_dac as f64) * 0.65).round() as u32; - if next < 1 { - self.finish_frequency(false); - } else { - self.current_dac = next.max(1); - } - } else { - // Drive up toward the boundary. - let next = ((measurement.amplitude_dac as f64) * 1.5).ceil() as u32; - if next > self.plan.max_amplitude_dac { - // Even full drive shows nothing: unmeasurable point. - self.finish_frequency(true); - } else { - self.current_dac = next; - } - } - } - Phase::Grid { .. } => { - self.advance_grid(); - } - } - self.current_command() - } - - fn bracketed(&self) -> bool { - matches!( - (self.highest_undetected, self.lowest_detected), - (Some(_), Some(_)) - ) - } - - fn grid_queue(&self) -> Vec { - let (Some(low), Some(high)) = (self.highest_undetected, self.lowest_detected) else { - return Vec::new(); - }; - let lo = (low.min(high) as f64 * 0.8).max(1.0); - let hi = (low.max(high) as f64 * 1.25).min(self.plan.max_amplitude_dac as f64); - let n = self.plan.grid_points.max(2); - (0..n) - .map(|i| { - let t = i as f64 / (n - 1) as f64; - (lo * (hi / lo).powf(t)).round() as u32 - }) - .collect() - } - - fn advance_grid(&mut self) { - let next = match &mut self.phase { - Phase::Grid { queue } if !queue.is_empty() => Some(queue.remove(0)), - _ => None, - }; - match next { - Some(dac) => self.current_dac = dac, - None => self.finish_frequency(false), - } - } - - fn finish_frequency(&mut self, exhausted: bool) { - let frequency_hz = self.plan.frequencies_hz[self.frequency_index]; - let fit = if exhausted { - None - } else { - fit_min_depth(&self.points) - }; - self.results.push(FrequencyResult { - frequency_hz, - fit, - points: std::mem::take(&mut self.points), - measurements: self.measurements_at_frequency, - exhausted, - }); - self.frequency_index += 1; - self.phase = Phase::Bisecting; - self.current_dac = self.plan.initial_amplitude_dac; - self.measurements_at_frequency = 0; - self.highest_undetected = None; - self.lowest_detected = None; - self.invalid_retries = 0; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Simulated bench: optical contrast is proportional to the drive code - /// (a = dac / 2000) and the pixel responds with the smeared first step - /// around a_min = 0.2. - fn respond(dac: u32) -> Measurement { - let a = dac as f64 / 2_000.0; - let z = (a.ln() - 0.2_f64.ln()) / 0.12; - let n = 0.5 * (1.0 + erf_approx(z / std::f64::consts::SQRT_2)); - Measurement { - amplitude_dac: dac, - measured_a: Some(a), - events_per_half_cycle: n, - detected: n > 0.15, - } - } - - fn erf_approx(x: f64) -> f64 { - let t = 1.0 / (1.0 + 0.327_591_1 * x.abs()); - let poly = t - * (0.254_829_592 - + t * (-0.284_496_736 - + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429)))); - let value = 1.0 - poly * (-x * x).exp(); - if x >= 0.0 { - value - } else { - -value - } - } - - #[test] - fn converges_to_the_synthetic_a_min() { - let mut engine = SweepEngine::new(SweepPlan { - frequencies_hz: vec![1_000.0, 10_000.0], - ..SweepPlan::default() - }); - - let mut guard = 0; - loop { - guard += 1; - assert!(guard < 200, "sweep must terminate"); - match engine.current_command() { - SweepCommand::Finished => break, - SweepCommand::Measure { amplitude_dac, .. } => { - engine.ingest(respond(amplitude_dac)); - } - } - } - - assert_eq!(engine.results.len(), 2); - for result in &engine.results { - let fit = result.fit.as_ref().expect("fit must exist"); - assert!( - (fit.a_min - 0.2).abs() < 0.04, - "f={} a_min={}", - result.frequency_hz, - fit.a_min - ); - assert!(!result.exhausted); - } - } - - #[test] - fn undetectable_frequency_is_reported_exhausted_not_fitted() { - let mut engine = SweepEngine::new(SweepPlan { - frequencies_hz: vec![100_000.0], - ..SweepPlan::default() - }); - let mut guard = 0; - loop { - guard += 1; - assert!(guard < 100); - match engine.current_command() { - SweepCommand::Finished => break, - SweepCommand::Measure { amplitude_dac, .. } => { - engine.ingest(Measurement { - amplitude_dac, - measured_a: Some(amplitude_dac as f64 / 2_000.0), - events_per_half_cycle: 0.0, - detected: false, - }); - } - } - } - assert_eq!(engine.results.len(), 1); - assert!(engine.results[0].exhausted); - assert!(engine.results[0].fit.is_none()); - } - - #[test] - fn invalid_windows_are_retried_then_abandoned() { - let mut engine = SweepEngine::new(SweepPlan { - frequencies_hz: vec![1_000.0], - ..SweepPlan::default() - }); - let mut measures = 0; - loop { - match engine.current_command() { - SweepCommand::Finished => break, - SweepCommand::Measure { amplitude_dac, .. } => { - measures += 1; - assert!(measures < 20); - engine.ingest(Measurement { - amplitude_dac, - measured_a: None, - events_per_half_cycle: 0.0, - detected: false, - }); - } - } - } - assert!(engine.results[0].exhausted); - } -} diff --git a/plugins/stage-a-funcgen/README.md b/plugins/stage-a-funcgen/README.md deleted file mode 100644 index fb57f6c..0000000 --- a/plugins/stage-a-funcgen/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Stage-A Function Generator - -Manual control of the Stage-A Pockels-cell drive for familiarisation with the -bench: pick a waveform (**sine**, **square**, **sawtooth**), a frequency, and a -DAC modulation depth, hit *Apply drive*, and watch the photodiode respond live. - -## Why the amplitude is "measured", not set - -`amplitude_dac` commands the *phase*-modulation depth of the Pockels cell. The -cell's voltage→transmission response is non-linear (≈ sin²), so the same DAC -excursion produces different optical amplitudes at different working points. -The plugin therefore always reports the **measured** optical log-contrast - -``` -a = ln(V_max / V_min) (dark-corrected photodiode voltages) -``` - -computed by `stage-a-io`'s calibrated, clipping-guarded estimator — never a -value inferred from the commanded DAC codes. - -## Ports - -| Port | Behaviour | -|---|---| -| `mock` (default) | Runs the waveform-extended mock controller in-process: full command round trip, synthetic photodiode stream through a Pockels-like sin² transfer. Zero hardware, zero risk. | -| `auto` / explicit device | Real Teensy over USB serial. Firmware 0.2.0 has **no waveform backend** and rejects the drive fields (`unknown_config_field`); the plugin reports this clearly. Real drive control needs the future v2 DDS firmware (`stage-a-controller/docs/features/waveform-drive.md`), which is blocked on the hardware freeze. | - -## Views and actions - -- **FuncGen photodiode** — live decimated waveform (volts vs. ms). -- **Function generator** status table — state, firmware, waveform-backend - capability, commanded drive, measured `a`, clipping, stream integrity. -- Actions on the status table: *Connect*, *Disconnect*, *Apply drive*, - *Stop drive*. - -## Safety model - -Same contract as `stage-a-monitor`: - -- serial/mock connections open only while the execution context is - `LiveCapture` with effects allowed — replay can never drive hardware; -- waveform/frequency/amplitude are persistent *settings*, but nothing reaches - the controller until the explicit *Apply drive* **action**; -- drives whose `center ± amplitude` leave the 0–4095 DAC range are refused - locally before any command is sent; -- `process_frame()` only drains the bounded I/O worker queues; -- watchdog `!FAULT` notices from the controller are surfaced immediately. diff --git a/plugins/stage-a-funcgen/plugin.toml b/plugins/stage-a-funcgen/plugin.toml deleted file mode 100644 index df29231..0000000 --- a/plugins/stage-a-funcgen/plugin.toml +++ /dev/null @@ -1,7 +0,0 @@ -name = "Stage-A Function Generator" -version = "0.2.0" -description = "Manual Pockels-cell drive control (sine/square/sawtooth, frequency, DAC amplitude) with the resulting optical contrast always measured from the photodiode." -domain = "stage-a" -library = "augur_plugin_stage_a_funcgen" -phase = "frame_only" -min_augur_version = "1.0.0" diff --git a/plugins/stage-a-funcgen/src/lib.rs b/plugins/stage-a-funcgen/src/lib.rs deleted file mode 100644 index 694a21c..0000000 --- a/plugins/stage-a-funcgen/src/lib.rs +++ /dev/null @@ -1,1011 +0,0 @@ -//! Stage-A function generator — familiarisation plugin. -//! -//! Manual control of the Pockels-cell drive: waveform (sine, square, -//! sawtooth), frequency, and the commanded DAC modulation depth -//! (`amplitude_dac`). The commanded amplitude sets the *phase* modulation -//! of the Pockels cell, which maps non-linearly to transmitted intensity — -//! so the optical amplitude shown here is always the photodiode-measured -//! log-contrast `a = ln(V_max/V_min)`, never the DAC excursion. -//! -//! Firmware 0.2.0 has no waveform backend yet: it rejects the reserved v2 -//! drive fields with `unknown_config_field` (the feature-detection -//! contract, `stage-a-controller/docs/features/waveform-drive.md`). Until -//! the DDS firmware lands, select the **`mock`** port: it runs the -//! waveform-extended mock controller in-process and streams a synthetic -//! photodiode response through a Pockels-like sin² transfer — the full -//! control loop with zero hardware and zero risk. -//! -//! Safety contract (same as `stage-a-monitor`): -//! - devices open only when the execution context is `LiveCapture` with -//! `effects_allowed`; anything else tears the connection down; -//! - drive parameters are persistent *settings*, but nothing starts the -//! hardware except an explicit Apply **action**; -//! - `process_frame()` only drains the bounded I/O worker queues. - -use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread::JoinHandle; -use std::time::{Duration, Instant}; - -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, - Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, - StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, -}; -use serde_json::{json, Value}; -use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, - MockController, MockState, StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, -}; - -const WAVEFORM_DATASET_ID: &str = "stage-a-funcgen.waveform"; -const STATUS_DATASET_ID: &str = "stage-a-funcgen.status"; -const WAVEFORM_VIEW_ID: &str = "stage-a-funcgen.waveform.view"; -const STATUS_VIEW_ID: &str = "stage-a-funcgen.status.view"; - -const ACTION_CONNECT: &str = "stage-a-funcgen.connect"; -const ACTION_DISCONNECT: &str = "stage-a-funcgen.disconnect"; -const ACTION_APPLY: &str = "stage-a-funcgen.apply"; -const ACTION_STOP: &str = "stage-a-funcgen.stop"; - -/// Retained sample window for the live view + contrast estimate. -const SAMPLE_RING_CAPACITY: usize = 32_768; -/// Points published per waveform refresh (decimated). -const WAVEFORM_POINTS: usize = 1_024; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ConnectionState { - Disconnected, - Connected, - Driving, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Wave { - Sine, - Square, - Saw, -} - -impl Wave { - const VARIANTS: [Wave; 3] = [Wave::Sine, Wave::Square, Wave::Saw]; - - fn name(self) -> &'static str { - match self { - Self::Sine => "SINE", - Self::Square => "SQUARE", - Self::Saw => "SAW", - } - } - - fn from_name(name: &str) -> Option { - Self::VARIANTS.into_iter().find(|w| w.name() == name) - } -} - -/// In-process mock controller thread behind the `mock` port. -struct MockService { - stop: Arc, - join: Option>, -} - -impl MockService { - fn spawn() -> (Self, StageAClient) { - let link = stage_a_io::MockLink::new(); - let stop = Arc::new(AtomicBool::new(false)); - let thread_stop = Arc::clone(&stop); - let mut controller = MockController::new(link.device_end()).with_waveform_extension(); - let join = std::thread::Builder::new() - .name("stage-a-funcgen-mock".into()) - .spawn(move || { - let mut last_block = Instant::now(); - while !thread_stop.load(Ordering::Relaxed) { - controller.poll_commands(); - if controller.state() == MockState::Running - && last_block.elapsed() >= controller.block_period() - { - last_block = Instant::now(); - controller.emit_configured_block(); - } - std::thread::sleep(Duration::from_millis(1)); - } - }) - .expect("spawning the mock controller thread must succeed"); - ( - Self { - stop, - join: Some(join), - }, - StageAClient::new(link.host_end()), - ) - } -} - -impl Drop for MockService { - fn drop(&mut self) { - self.stop.store(true, Ordering::Relaxed); - if let Some(join) = self.join.take() { - let _ = join.join(); - } - } -} - -pub struct StageAFuncGenPlugin { - enabled: bool, - // -- device -- - worker: Option, - mock_service: Option, - connection: ConnectionState, - firmware: String, - has_waveform_backend: Option, - next_tag: u64, - in_flight: BTreeMap, - last_error: Option, - integrity: StreamIntegrity, - effects_blocked_reason: Option, - // -- settings (drive parameters; applying them is an explicit action) -- - port_hint: String, - wave: Wave, - frequency_hz: f64, - center_dac: i64, - amplitude_dac: i64, - sample_rate_hz: i64, - calibration: AdcCalibration, - // -- data -- - sample_ring: Vec, - ring_next_sample_index: u64, - sample_rate_seen_hz: u32, - contrast: Option, - contrast_error: Option, - dataset_generation: u64, - consumed_action_ids: Vec, -} - -impl Default for StageAFuncGenPlugin { - fn default() -> Self { - Self { - enabled: false, - worker: None, - mock_service: None, - connection: ConnectionState::Disconnected, - firmware: String::new(), - has_waveform_backend: None, - next_tag: 1, - in_flight: BTreeMap::new(), - last_error: None, - integrity: StreamIntegrity::default(), - effects_blocked_reason: None, - port_hint: "mock".into(), - wave: Wave::Sine, - frequency_hz: 1_000.0, - center_dac: 2_048, - amplitude_dac: 512, - sample_rate_hz: 20_000, - calibration: AdcCalibration::default(), - sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), - ring_next_sample_index: 0, - sample_rate_seen_hz: 0, - contrast: None, - contrast_error: None, - dataset_generation: 0, - consumed_action_ids: Vec::new(), - } - } -} - -impl StageAFuncGenPlugin { - fn bump_generation(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - - fn connect(&mut self) { - if self.worker.is_some() { - return; - } - if self.port_hint == "mock" { - let (service, client) = MockService::spawn(); - self.mock_service = Some(service); - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - } else { - match open_serial(&self.port_hint) { - Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - } - Err(err) => self.last_error = Some(err), - } - } - self.bump_generation(); - } - - fn disconnect(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - // Shut the worker down first: its final STOP still needs the - // mock service (if any) alive to be acknowledged. - worker.shutdown(reason); - } - self.mock_service = None; - self.connection = ConnectionState::Disconnected; - self.firmware.clear(); - self.has_waveform_backend = None; - self.in_flight.clear(); - self.bump_generation(); - } - - /// STOP → CONFIG (drive fields) → START, honouring the firmware state - /// machine (CONFIG is only legal from SAFE_IDLE/CONFIGURED). - fn apply_drive(&mut self) { - let center = self.center_dac.clamp(0, 4_095); - let amplitude = self.amplitude_dac.clamp(0, 2_047); - if center + amplitude > 4_095 || amplitude > center { - self.last_error = Some(format!( - "drive: center {center} ± amplitude {amplitude} exceeds the 0–4095 DAC range" - )); - return; - } - let freq_mhz = ((self.frequency_hz.max(0.001)) * 1_000.0).round() as i64; - self.queue_command("stop", Command::new("STOP").field("reason", "reconfigure")); - self.queue_command( - "drive", - Command::new("CONFIG") - .field("mode", "A1") - .field("wave", self.wave.name()) - .field("freq_mhz", freq_mhz) - .field("center_dac", center) - .field("amplitude_dac", amplitude) - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", 256) - .field("raw", 1) - .field("summary", 1), - ); - self.queue_command("start", Command::new("START")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(true)); - } - } - - fn stop_drive(&mut self) { - self.queue_command("stop", Command::new("STOP").field("reason", "operator")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - if outputs.is_empty() { - return; - } - let mut changed = false; - let mut stopped: Option = None; - for output in outputs { - changed = true; - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => self.handle_reply(&purpose, &fields), - Err(err) if err.contains("unknown_config_field") => { - self.has_waveform_backend = Some(false); - self.last_error = Some( - "firmware has no waveform backend (v1) — select the mock port \ - or wait for the v2 DDS firmware" - .into(), - ); - } - Err(err) => { - self.last_error = Some(format!("{purpose}: {err}")); - } - } - } - WorkerOutput::Event(DeviceEvent::Data(frame)) => { - if frame.header.frame_type == FrameType::SamplesU16 { - if let Some(codes) = frame.samples() { - self.sample_rate_seen_hz = frame.header.sample_rate_hz; - self.push_samples(&codes, frame.header.first_sample_index); - } - } - } - WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { - if name == "FAULT" { - if self.connection == ConnectionState::Driving { - self.connection = ConnectionState::Connected; - } - self.last_error = Some(format!( - "controller fault: {} — dropped to SAFE_IDLE", - fields.get("code").map(String::as_str).unwrap_or("unknown") - )); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - } - WorkerOutput::Integrity(integrity) => { - self.integrity = integrity; - } - WorkerOutput::Stopped { reason } => { - stopped = Some(reason); - } - } - } - if let Some(reason) = stopped { - self.worker = None; - self.mock_service = None; - self.connection = ConnectionState::Disconnected; - self.last_error = Some(format!("device connection ended: {reason}")); - } - if changed { - self.refresh_contrast(); - self.bump_generation(); - } - } - - fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { - match purpose { - "hello" => { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - self.has_waveform_backend = Some( - fields - .get("capabilities") - .is_some_and(|caps| caps.split(',').any(|c| c == "WAVE")), - ); - self.connection = ConnectionState::Connected; - } - "drive" => { - self.has_waveform_backend = Some(true); - } - "start" => { - self.connection = ConnectionState::Driving; - } - "stop" => { - if self.connection == ConnectionState::Driving { - self.connection = ConnectionState::Connected; - } - } - _ => {} - } - } - - fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { - self.ring_next_sample_index = first_sample_index + codes.len() as u64; - self.sample_ring.extend_from_slice(codes); - let len = self.sample_ring.len(); - if len > SAMPLE_RING_CAPACITY { - self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); - } - } - - fn refresh_contrast(&mut self) { - if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { - return; - } - match estimate_contrast(&self.sample_ring, &self.calibration) { - Ok(estimate) => { - self.contrast = Some(estimate); - self.contrast_error = None; - } - Err(err) => { - self.contrast = None; - self.contrast_error = Some(err.to_string()); - } - } - } - - fn waveform_dataset(&self) -> Series1dV1 { - let rate = if self.sample_rate_seen_hz > 0 { - f64::from(self.sample_rate_seen_hz) - } else { - self.sample_rate_hz as f64 - }; - let n = self.sample_ring.len(); - let stride = (n / WAVEFORM_POINTS).max(1); - let first_index = self.ring_next_sample_index.saturating_sub(n as u64); - let points: Vec = self - .sample_ring - .iter() - .enumerate() - .step_by(stride) - .map(|(i, &code)| Series1dPoint { - x: (first_index + i as u64) as f64 / rate * 1_000.0, - y: self.calibration.code_to_volts(code), - }) - .collect(); - Series1dV1 { - x_label: "time [ms]".into(), - y_label: "photodiode [V]".into(), - lines: vec![Series1dLine { - name: "photodiode".into(), - points, - }], - } - } - - fn drive_summary(&self) -> String { - format!( - "{} @ {:.3} Hz, {} ± {} DAC", - self.wave.name(), - self.frequency_hz, - self.center_dac, - self.amplitude_dac - ) - } - - fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.connection) { - (Some(reason), _) => format!("locked ({reason})"), - (None, ConnectionState::Disconnected) => "disconnected".into(), - (None, ConnectionState::Connected) => "connected".into(), - (None, ConnectionState::Driving) => "driving".into(), - }; - let backend = match self.has_waveform_backend { - Some(true) => "waveform-capable".into(), - Some(false) => "no waveform backend (v1)".into(), - None => "—".into(), - }; - let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { - (Some(estimate), _) => ( - format!("{:.4}", estimate.a), - format!( - "{:.2}% low / {:.2}% high", - estimate.low_clip_fraction * 100.0, - estimate.high_clip_fraction * 100.0 - ), - ), - (None, Some(err)) => ("invalid".into(), err.clone()), - (None, None) => ("—".into(), "—".into()), - }; - let integrity = if self.integrity.is_clean() { - "clean".to_owned() - } else { - format!( - "crc={} gaps={} skipped={} overruns={}", - self.integrity.crc_failures, - self.integrity.sequence_gaps, - self.integrity.skipped_bytes, - self.integrity.dropped_samples - ) - }; - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", state), - text_column("firmware", self.firmware.clone()), - text_column("backend", backend), - text_column("drive", self.drive_summary()), - text_column("a", a_text), - text_column("clipping", clip_text), - text_column("integrity", integrity), - text_column("error", self.last_error.clone().unwrap_or_default()), - ], - } - } - - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("firmware", "Firmware"), - column("backend", "Waveform backend"), - column("drive", "Commanded drive"), - column("a", "Measured a = ln(Vmax/Vmin)"), - column("clipping", "Clipping"), - column("integrity", "Stream integrity"), - column("error", "Last error"), - ], - ..TableSchema::default() - } - } - - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-funcgen.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed - } -} - -fn open_serial(port_hint: &str) -> Result, String> { - let path = if port_hint == "auto" { - serial_ports() - .into_iter() - .next() - .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? - } else { - port_hint.to_owned() - }; - let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} - -fn serial_ports() -> Vec { - stage_a_io::transport::available_port_names() - .into_iter() - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) - .collect() -} - -impl Plugin for StageAFuncGenPlugin { - fn name(&self) -> &'static str { - "Stage-A Function Generator" - } - - fn description(&self) -> &'static str { - "Manual Pockels-cell drive (sine/square/sawtooth, frequency, DAC amplitude) with photodiode-measured optical contrast; mock port for hardware-free familiarisation." - } - - fn enabled(&self) -> bool { - self.enabled - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.disconnect("plugin disabled"); - } - } - - fn reset(&mut self) { - self.sample_ring.clear(); - self.contrast = None; - self.contrast_error = None; - self.bump_generation(); - } - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - // Fail closed: any pass without live-capture effects tears the - // connection down and refuses commands — even for the mock port, - // so switching the port setting can never bypass the gate. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.worker.is_some() { - self.disconnect("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for action_id in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect("operator"), - ACTION_APPLY => self.apply_drive(), - ACTION_STOP => self.stop_drive(), - _ => {} - } - } - - self.drain_worker(); - } - - fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; - port_variants.extend(serial_ports()); - let port_default = port_variants - .iter() - .position(|p| *p == self.port_hint) - .unwrap_or(0); - let wave_variants: Vec = - Wave::VARIANTS.iter().map(|w| w.name().to_owned()).collect(); - let wave_default = Wave::VARIANTS - .iter() - .position(|w| *w == self.wave) - .unwrap_or(0); - SettingsSchema { - sections: vec![SettingsSection { - label: "Function generator".into(), - description: Some( - "Drive parameters are settings; nothing reaches the hardware until the \ - Apply action. The optical amplitude is measured from the photodiode — \ - the DAC amplitude is a phase-modulation depth, not a light level." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "mock = in-process simulated controller (no hardware); \ - auto = first Teensy USB serial device" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, - }, - }, - SettingItem { - key: "wave".into(), - label: "Waveform".into(), - tooltip: Some("SINE, SQUARE, or SAW (sawtooth / Sägezahn)".into()), - kind: SettingKind::Enum { - variants: wave_variants, - default: wave_default, - }, - }, - SettingItem { - key: "frequency_hz".into(), - label: "Frequency".into(), - tooltip: Some("Drive frequency (sent as integer millihertz)".into()), - kind: SettingKind::F64Drag { - min: 0.001, - max: 200_000.0, - speed: 1.0, - default: self.frequency_hz, - }, - }, - SettingItem { - key: "center_dac".into(), - label: "Center DAC code".into(), - tooltip: Some("Working-point code (0–4095)".into()), - kind: SettingKind::I64Drag { - min: 0, - max: 4_095, - default: self.center_dac, - }, - }, - SettingItem { - key: "amplitude_dac".into(), - label: "Amplitude DAC code".into(), - tooltip: Some( - "Pockels phase-modulation depth (0–2047); the optical contrast \ - this produces is read from the measured a" - .into(), - ), - kind: SettingKind::I64Drag { - min: 0, - max: 2_047, - default: self.amplitude_dac, - }, - }, - SettingItem { - key: "sample_rate_hz".into(), - label: "ADC sample rate".into(), - tooltip: Some("Photodiode sample rate for the feedback stream".into()), - kind: SettingKind::I64Slider { - min: 1_000, - max: 100_000, - default: self.sample_rate_hz, - suffix: Some(" Hz".into()), - }, - }, - SettingItem { - key: "dark_millivolts".into(), - label: "Dark level".into(), - tooltip: Some( - "Light-blocked photodiode level; a is computed from dark-corrected \ - voltages" - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 3_300.0, - speed: 1.0, - default: self.calibration.dark_volts * 1_000.0, - }, - }, - ], - }], - } - } - - fn get_setting(&self, key: &str) -> Option { - match key { - "port" => Some(json!(self.port_hint)), - "wave" => Some(json!(self.wave.name())), - "frequency_hz" => Some(json!(self.frequency_hz)), - "center_dac" => Some(json!(self.center_dac)), - "amplitude_dac" => Some(json!(self.amplitude_dac)), - "sample_rate_hz" => Some(json!(self.sample_rate_hz)), - "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); - Ok(()) - } - "wave" => { - let name = value.as_str().ok_or("wave must be a string")?; - self.wave = Wave::from_name(name) - .ok_or_else(|| format!("unknown waveform: {name} (SINE/SQUARE/SAW)"))?; - Ok(()) - } - "frequency_hz" => { - let hz = value.as_f64().ok_or("frequency_hz must be a number")?; - self.frequency_hz = hz.clamp(0.001, 200_000.0); - Ok(()) - } - "center_dac" => { - self.center_dac = value - .as_i64() - .ok_or("center_dac must be an integer")? - .clamp(0, 4_095); - Ok(()) - } - "amplitude_dac" => { - self.amplitude_dac = value - .as_i64() - .ok_or("amplitude_dac must be an integer")? - .clamp(0, 2_047); - Ok(()) - } - "sample_rate_hz" => { - self.sample_rate_hz = value - .as_i64() - .ok_or("sample_rate_hz must be an integer")? - .clamp(1_000, 100_000); - Ok(()) - } - "dark_millivolts" => { - let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; - self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); - Ok(()) - } - _ => Err(format!("unknown setting: {key}")), - } - } - - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - entries.push(StatusEntry::Text(match self.connection { - ConnectionState::Disconnected => "FuncGen: disconnected".into(), - ConnectionState::Connected => format!("FuncGen: connected ({})", self.firmware), - ConnectionState::Driving => format!("FuncGen: driving {}", self.drive_summary()), - })); - if let Some(estimate) = &self.contrast { - entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); - } - entries - } - - fn host_views(&self) -> HostViewRegistry { - let dataset_action = |id: &str, title: &str| HostActionDescriptor { - id: id.into(), - title: title.into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }; - HostViewRegistry { - datasets: vec![ - HostDatasetDescriptor { - id: WAVEFORM_DATASET_ID.into(), - title: "FuncGen photodiode waveform".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No photodiode samples yet — connect and apply a drive.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "Function generator status".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Function generator idle.".into(), - display: None, - relations: Vec::new(), - }, - ], - views: vec![ - HostViewDescriptor { - id: WAVEFORM_VIEW_ID.into(), - title: "FuncGen photodiode".into(), - dataset_id: WAVEFORM_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: STATUS_VIEW_ID.into(), - title: "Function generator".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - ], - actions: vec![ - dataset_action(ACTION_CONNECT, "Connect"), - dataset_action(ACTION_DISCONNECT, "Disconnect"), - dataset_action(ACTION_APPLY, "Apply drive"), - dataset_action(ACTION_STOP, "Stop drive"), - ], - } - } - - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, - } - } - - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), - _ => 0, - } - } -} - -impl Drop for StageAFuncGenPlugin { - fn drop(&mut self) { - self.disconnect("plugin destroyed"); - } -} - -export_plugin!(StageAFuncGenPlugin); - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{Duration, Instant}; - - fn drain_until bool>( - plugin: &mut StageAFuncGenPlugin, - timeout: Duration, - mut done: F, - ) { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - plugin.drain_worker(); - if done(plugin) { - return; - } - std::thread::sleep(Duration::from_millis(2)); - } - panic!("condition not reached within {timeout:?}"); - } - - /// Full mock loop: connect → apply sine → measured a appears → stop. - #[test] - fn mock_port_round_trip_measures_optical_contrast() { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.calibration.dark_volts = 40.0 * 3.3 / 4_095.0; - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - assert_eq!(plugin.has_waveform_backend, Some(true)); - assert_eq!(plugin.firmware, "0.2.0-mock"); - - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving && p.contrast.is_some() - }); - let a = plugin.contrast.as_ref().expect("contrast measured").a; - assert!(a > 0.0, "modulated drive must produce positive contrast"); - assert!(plugin.integrity.is_clean()); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - - plugin.stop_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - plugin.disconnect("test done"); - assert_eq!(plugin.connection, ConnectionState::Disconnected); - } - - /// Square and sawtooth are accepted and produce a measurable contrast. - #[test] - fn square_and_saw_waveforms_drive_the_mock() { - for wave in [Wave::Square, Wave::Saw] { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.wave = wave; - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving && p.contrast.is_some() - }); - assert!(plugin.contrast.as_ref().unwrap().a > 0.0); - plugin.disconnect("done"); - } - } - - /// Drives exceeding the DAC range are refused locally, before any - /// command reaches a controller. - #[test] - fn out_of_range_drive_is_rejected_locally() { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.center_dac = 3_000; - plugin.amplitude_dac = 2_000; - plugin.apply_drive(); - assert!(plugin - .last_error - .as_deref() - .is_some_and(|err| err.contains("exceeds the 0–4095 DAC range"))); - } - - /// Re-applying while driving must STOP first (firmware state machine). - #[test] - fn reapply_while_driving_reconfigures_cleanly() { - let mut plugin = StageAFuncGenPlugin::default(); - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Connected - }); - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving - }); - plugin.frequency_hz = 2_000.0; - plugin.apply_drive(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.connection == ConnectionState::Driving && p.last_error.is_none() - }); - plugin.disconnect("done"); - } -} diff --git a/plugins/stage-a-funcgen/Cargo.toml b/plugins/stage-a-modulation/Cargo.toml similarity index 58% rename from plugins/stage-a-funcgen/Cargo.toml rename to plugins/stage-a-modulation/Cargo.toml index ffb540e..2b04e93 100644 --- a/plugins/stage-a-funcgen/Cargo.toml +++ b/plugins/stage-a-modulation/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "augur-plugin-stage-a-funcgen" +name = "augur-plugin-stage-a-modulation" version.workspace = true edition.workspace = true license.workspace = true authors.workspace = true -description = "Stage-A function generator: manual waveform/frequency/amplitude drive control with photodiode-measured optical contrast." +description = "Stage-A laser modulation control: one capped power slider plus constant/sine/square drive of the Teensy DAC (J23), applied immediately." [lib] crate-type = ["cdylib", "rlib"] diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md new file mode 100644 index 0000000..9e21a88 --- /dev/null +++ b/plugins/stage-a-modulation/README.md @@ -0,0 +1,33 @@ +# Stage-A Modulation + +Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy **command port** +(the first of the two USB serial ports enumerated by `stage-a-controller` firmware 0.3.0+). + +## What it does + +- **Power slider** in DAC codes (0–4095). Its upper bound is the **max limit** setting — set that + to the highest code the connected device tolerates and the slider physically cannot exceed it. +- **Mode**: `CONST` (hold the level), `SINE`, or `SQUARE` with a **frequency** (0.01–2000 Hz) and + a **min threshold** — the periodic waveforms swing between the threshold and the slider value. +- Every accepted change is sent to the Teensy **immediately** (one `MOD` command); there is no + Apply button. +- The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and + a 2 Hz `STATUS` poll), not just what was commanded. + +## Actions + +- **Connect / Disconnect** — open/close the command port. Connecting never changes the output; + only changes made while connected are transferred. +- **Output OFF** — sends `MOD wave=OFF` (DAC code 0). Needed because the firmware output is + **set-and-hold**: disconnecting, closing the GUI, or a crash leaves the last modulation running + (`stage-a-controller` ADR 002). + +## Ports + +Select the Teensy *command* port (binary protocol), not the photodiode stream port. `mock` runs an +in-process simulated controller for hardware-free testing; `auto` picks the first +usbmodem/ttyACM device. If you picked the wrong physical port, HELLO simply times out — pick the +other one. + +Hardware commands only flow while the host execution context allows effects (live capture); +otherwise the connection is torn down and the panel shows the lock reason. diff --git a/plugins/stage-a-modulation/plugin.toml b/plugins/stage-a-modulation/plugin.toml new file mode 100644 index 0000000..39b9d48 --- /dev/null +++ b/plugins/stage-a-modulation/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Modulation" +version = "0.3.0" +description = "Laser modulation control: capped power slider plus constant/sine/square drive of the Teensy DAC (J23), applied immediately." +domain = "stage-a" +library = "augur_plugin_stage_a_modulation" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs new file mode 100644 index 0000000..e5b163a --- /dev/null +++ b/plugins/stage-a-modulation/src/lib.rs @@ -0,0 +1,844 @@ +//! Stage-A laser modulation control. +//! +//! Drives the laser modulation input (Hermit J23, `DAC1.4`/address 3) through +//! the firmware 0.3.0 `MOD` command. One power slider (DAC code) whose upper +//! bound is a user-set safety cap, a mode select (constant / sine / square) +//! with frequency and a lower threshold for the periodic modes — and every +//! accepted change is transferred to the Teensy immediately, no Apply button. +//! +//! The plugin owns the Teensy **command port** (the first of the two CDC +//! ports the dual-serial firmware enumerates; the photodiode stream port is +//! owned by `stage-a-photodiode`). The firmware output is set-and-hold: +//! disconnecting does NOT switch the modulation off — use the "Output OFF" +//! action (ADR 002 in `stage-a-controller`). +//! +//! Safety contract: +//! - devices open only when the execution context allows hardware effects; +//! anything else tears the connection down (fail closed); +//! - the level slider cannot exceed the max-level cap, and the firmware +//! output can never exceed the slider (square/sine peak at `level`); +//! - `process_frame()` only drains the bounded I/O worker queues. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, SettingItem, + SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, + TableColumnValues, TableDatasetV1, TableSchema, TableValueType, + CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; +use stage_a_io::{Command, IoWorker, MockController, StageAClient, WorkerOutput, WorkerRequest}; + +const STATUS_DATASET_ID: &str = "stage-a-modulation.status"; +const STATUS_VIEW_ID: &str = "stage-a-modulation.status.view"; + +const ACTION_CONNECT: &str = "stage-a-modulation.connect"; +const ACTION_DISCONNECT: &str = "stage-a-modulation.disconnect"; +const ACTION_OUTPUT_OFF: &str = "stage-a-modulation.output-off"; + +const MAX_DAC_CODE: i64 = 4_095; +const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Const, + Sine, + Square, +} + +impl Mode { + const VARIANTS: [Mode; 3] = [Mode::Const, Mode::Sine, Mode::Square]; + + fn name(self) -> &'static str { + match self { + Self::Const => "CONST", + Self::Sine => "SINE", + Self::Square => "SQUARE", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|m| m.name() == name) + } + + fn is_periodic(self) -> bool { + !matches!(self, Self::Const) + } +} + +/// In-process mock controller thread behind the `mock` port. +struct MockService { + stop: Arc, + join: Option>, +} + +impl MockService { + fn spawn() -> (Self, StageAClient) { + let link = stage_a_io::MockLink::new(); + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let mut controller = MockController::new(link.device_end()); + let join = std::thread::Builder::new() + .name("stage-a-modulation-mock".into()) + .spawn(move || { + while !thread_stop.load(Ordering::Relaxed) { + controller.poll_commands(); + std::thread::sleep(Duration::from_millis(1)); + } + }) + .expect("spawning the mock controller thread must succeed"); + ( + Self { + stop, + join: Some(join), + }, + StageAClient::new(link.host_end()), + ) + } +} + +impl Drop for MockService { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +pub struct StageAModulationPlugin { + enabled: bool, + // -- device -- + worker: Option, + mock_service: Option, + connected: bool, + firmware: String, + next_tag: u64, + in_flight: BTreeMap, + last_error: Option, + effects_blocked_reason: Option, + last_status_poll: Instant, + // -- settings (every accepted change is sent immediately) -- + port_hint: String, + max_level: i64, + level: i64, + min_level: i64, + mode: Mode, + frequency_hz: f64, + dirty: bool, + // -- board-reported state (from MOD replies and STATUS polls) -- + board_code: Option, + board_mod: String, + dataset_generation: u64, + consumed_action_ids: Vec, +} + +impl Default for StageAModulationPlugin { + fn default() -> Self { + Self { + enabled: false, + worker: None, + mock_service: None, + connected: false, + firmware: String::new(), + next_tag: 1, + in_flight: BTreeMap::new(), + last_error: None, + effects_blocked_reason: None, + last_status_poll: Instant::now(), + port_hint: "mock".into(), + max_level: MAX_DAC_CODE, + level: 0, + min_level: 0, + mode: Mode::Const, + frequency_hz: 10.0, + dirty: false, + board_code: None, + board_mod: "—".into(), + dataset_generation: 0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAModulationPlugin { + fn bump_generation(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + fn queue_command(&mut self, purpose: &str, command: Command) { + let Some(worker) = &self.worker else { + self.last_error = Some(format!("{purpose}: no device connection")); + return; + }; + let tag = self.next_tag; + self.next_tag += 1; + match worker.try_send(WorkerRequest::Send { tag, command }) { + Ok(()) => { + self.in_flight.insert(tag, purpose.to_owned()); + } + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + + fn connect(&mut self) { + if self.worker.is_some() { + return; + } + if self.port_hint == "mock" { + let (service, client) = MockService::spawn(); + self.mock_service = Some(service); + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + } else { + match open_serial(&self.port_hint) { + Ok(client) => { + self.worker = Some(IoWorker::spawn(client)); + self.last_error = None; + } + Err(err) => { + self.last_error = Some(err); + return; + } + } + } + // Connecting never drives the output: only changes made while + // connected are transferred. + self.dirty = false; + self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); + self.bump_generation(); + } + + fn disconnect(&mut self, reason: &str) { + if let Some(worker) = self.worker.take() { + worker.shutdown(reason); + } + self.mock_service = None; + self.connected = false; + self.firmware.clear(); + self.in_flight.clear(); + self.board_code = None; + self.board_mod = "—".into(); + self.bump_generation(); + } + + /// One MOD command carrying the complete current drive settings. + fn send_modulation(&mut self) { + self.dirty = false; + let level = self.level.clamp(0, self.max_level); + let mut command = Command::new("MOD") + .field("wave", self.mode.name()) + .field("level", level); + if self.mode.is_periodic() { + let freq_mhz = (self.frequency_hz.clamp(0.01, 2_000.0) * 1_000.0).round() as i64; + command = command + .field("min", self.min_level.clamp(0, level)) + .field("freq_mhz", freq_mhz); + } + self.queue_command("mod", command); + } + + fn output_off(&mut self) { + self.dirty = false; + self.queue_command("mod", Command::new("MOD").field("wave", "OFF")); + } + + fn drain_worker(&mut self) { + let Some(worker) = &self.worker else { + return; + }; + let outputs = worker.drain_outputs(); + if outputs.is_empty() { + return; + } + let mut stopped: Option = None; + for output in outputs { + match output { + WorkerOutput::Reply { tag, result } => { + let purpose = self.in_flight.remove(&tag).unwrap_or_default(); + match result { + Ok(fields) => self.handle_reply(&purpose, &fields), + Err(err) => self.last_error = Some(format!("{purpose}: {err}")), + } + } + WorkerOutput::Event(_) | WorkerOutput::Integrity(_) => {} + WorkerOutput::Stopped { reason } => stopped = Some(reason), + } + } + if let Some(reason) = stopped { + self.worker = None; + self.mock_service = None; + self.connected = false; + self.last_error = Some(format!("device connection ended: {reason}")); + } + self.bump_generation(); + } + + fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { + if purpose == "hello" { + self.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + self.connected = true; + let has_mod = fields + .get("capabilities") + .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); + if !has_mod { + self.last_error = + Some("firmware has no MOD capability — flash stage-a-controller 0.3.0+".into()); + } + } + // MOD replies and STATUS polls both carry code= and mod_* fields. + if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { + self.board_code = Some(code); + } + if let Some(wave) = fields.get("mod_wave") { + let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); + let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); + let freq_mhz = fields + .get("mod_freq_mhz") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + self.board_mod = if wave == "SINE" || wave == "SQUARE" { + format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) + } else { + format!("{wave} level={level}") + }; + } + if purpose == "mod" { + self.last_error = None; + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-modulation.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } + + fn commanded_summary(&self) -> String { + if self.mode.is_periodic() { + format!( + "{} {}..{} @ {:.3} Hz", + self.mode.name(), + self.min_level, + self.level, + self.frequency_hz + ) + } else { + format!("{} level={}", self.mode.name(), self.level) + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = match (&self.effects_blocked_reason, self.connected) { + (Some(reason), _) => format!("locked ({reason})"), + (None, false) => "disconnected".into(), + (None, true) => format!("connected ({})", self.firmware), + }; + let board_code = self + .board_code + .map_or_else(|| "—".into(), |code| code.to_string()); + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("commanded", self.commanded_summary()), + text_column("board_mod", self.board_mod.clone()), + text_column("board_code", board_code), + text_column("error", self.last_error.clone().unwrap_or_default()), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("commanded", "Commanded drive"), + column("board_mod", "Board modulation"), + column("board_code", "Board DAC code"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } +} + +fn open_serial(port_hint: &str) -> Result, String> { + let path = if port_hint == "auto" { + serial_ports() + .into_iter() + .next() + .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? + } else { + port_hint.to_owned() + }; + let transport = + stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +fn serial_ports() -> Vec { + stage_a_io::transport::available_port_names() + .into_iter() + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() +} + +impl Plugin for StageAModulationPlugin { + fn name(&self) -> &'static str { + "Stage-A Modulation" + } + + fn description(&self) -> &'static str { + "Laser modulation control on the Teensy command port: capped power slider, constant/sine/square with frequency, applied immediately; shows the DAC code the board reports." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect("plugin disabled"); + } + } + + fn reset(&mut self) { + self.bump_generation(); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Fail closed: without live-capture effects the connection is torn + // down and no command leaves the plugin. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.worker.is_some() { + self.disconnect("execution context revoked effects"); + } + return; + } + self.effects_blocked_reason = None; + + for action_id in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect("operator"), + ACTION_OUTPUT_OFF => self.output_off(), + _ => {} + } + } + + if self.dirty && self.connected { + self.send_modulation(); + } + if self.connected && self.last_status_poll.elapsed() >= STATUS_POLL_INTERVAL { + self.last_status_poll = Instant::now(); + self.queue_command("status", Command::new("STATUS")); + } + self.drain_worker(); + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + let mode_variants: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let mode_default = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Laser modulation".into(), + description: Some( + "Every change is sent to the Teensy immediately. The output never exceeds \ + the power slider, and the slider never exceeds the max limit. The firmware \ + holds the output when the plugin disconnects — use Output OFF to drive 0." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "Teensy command port (the FIRST of the two usbmodem ports); \ + mock = in-process simulated controller, auto = first device" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "level".into(), + label: "Power (DAC code)".into(), + tooltip: Some( + "Output level in DAC codes; peak value for sine/square. \ + Capped by the max limit below." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.level, + suffix: None, + }, + }, + SettingItem { + key: "max_level".into(), + label: "Max limit (DAC code)".into(), + tooltip: Some( + "Safety cap: the slider cannot go above this. Set it to the \ + highest code the connected device tolerates at J23." + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.max_level, + }, + }, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, + }, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.frequency_hz, + }, + }, + SettingItem { + key: "min_level".into(), + label: "Min threshold (DAC code)".into(), + tooltip: Some( + "Lower bound for sine/square: the waveform swings between this \ + and the power slider. Ignored in CONST mode." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.min_level, + suffix: None, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "level" => Some(json!(self.level)), + "max_level" => Some(json!(self.max_level)), + "mode" => Some(json!(self.mode.name())), + "frequency_hz" => Some(json!(self.frequency_hz)), + "min_level" => Some(json!(self.min_level)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "level" => { + self.level = value + .as_i64() + .ok_or("level must be an integer")? + .clamp(0, self.max_level); + if self.min_level > self.level { + self.min_level = self.level; + } + self.dirty = true; + Ok(()) + } + "max_level" => { + self.max_level = value + .as_i64() + .ok_or("max_level must be an integer")? + .clamp(0, MAX_DAC_CODE); + // Lowering the cap below the current level lowers the output. + if self.level > self.max_level { + self.level = self.max_level; + self.dirty = true; + } + if self.min_level > self.max_level { + self.min_level = self.max_level; + } + Ok(()) + } + "mode" => { + let name = value.as_str().ok_or("mode must be a string")?; + self.mode = Mode::from_name(name) + .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; + self.dirty = true; + Ok(()) + } + "frequency_hz" => { + let hz = value.as_f64().ok_or("frequency_hz must be a number")?; + self.frequency_hz = hz.clamp(0.01, 2_000.0); + if self.mode.is_periodic() { + self.dirty = true; + } + Ok(()) + } + "min_level" => { + self.min_level = value + .as_i64() + .ok_or("min_level must be an integer")? + .clamp(0, self.level); + if self.mode.is_periodic() { + self.dirty = true; + } + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + entries.push(StatusEntry::Text(if self.connected { + format!("Modulation: connected ({})", self.firmware) + } else { + "Modulation: disconnected".into() + })); + if let Some(code) = self.board_code { + entries.push(StatusEntry::Text(format!( + "Board: code={code} ({})", + self.board_mod + ))); + } + if let Some(error) = &self.last_error { + entries.push(StatusEntry::Text(format!("Error: {error}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + let action = |id: &str, title: &str| HostActionDescriptor { + id: id.into(), + title: title.into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }; + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Laser modulation".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Modulation control idle.".into(), + display: None, + relations: Vec::new(), + }], + views: vec![HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Laser modulation".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }], + actions: vec![ + action(ACTION_CONNECT, "Connect"), + action(ACTION_DISCONNECT, "Disconnect"), + action(ACTION_OUTPUT_OFF, "Output OFF"), + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + STATUS_DATASET_ID => self.dataset_generation.max(1), + _ => 0, + } + } +} + +impl Drop for StageAModulationPlugin { + fn drop(&mut self) { + self.disconnect("plugin destroyed"); + } +} + +export_plugin!(StageAModulationPlugin); + +#[cfg(test)] +mod tests { + use super::*; + + fn drain_until bool>( + plugin: &mut StageAModulationPlugin, + timeout: Duration, + mut done: F, + ) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + plugin.drain_worker(); + if done(plugin) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("condition not reached within {timeout:?}"); + } + + /// Slider change → MOD sent immediately → board echoes the code. + #[test] + fn level_change_transfers_immediately_and_board_code_is_shown() { + let mut plugin = StageAModulationPlugin::default(); + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); + assert_eq!(plugin.firmware, "0.3.0-mock"); + + plugin + .set_setting("level", json!(1234)) + .expect("level accepted"); + assert!(plugin.dirty); + plugin.send_modulation(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.board_code == Some(1234) + }); + assert!(!plugin.dirty); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + plugin.disconnect("test done"); + } + + /// The max cap bounds the slider, and lowering it re-sends a lower level. + #[test] + fn max_level_caps_the_slider() { + let mut plugin = StageAModulationPlugin::default(); + plugin.set_setting("max_level", json!(1000)).unwrap(); + plugin.set_setting("level", json!(4095)).unwrap(); + assert_eq!(plugin.level, 1000, "slider clamps to the cap"); + + plugin.set_setting("max_level", json!(500)).unwrap(); + assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); + assert!(plugin.dirty, "the lowered level must be transferred"); + + let schema = plugin.settings_schema(); + let level_item = schema.sections[0] + .items + .iter() + .find(|item| item.key == "level") + .expect("level setting exists"); + match &level_item.kind { + SettingKind::I64Slider { max, .. } => assert_eq!(*max, 500), + other => panic!("level must stay a slider, got {other:?}"), + } + } + + /// Square drive with min threshold reaches the mock and starts at min. + #[test] + fn square_with_min_threshold_round_trips() { + let mut plugin = StageAModulationPlugin::default(); + plugin.connect(); + drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); + + plugin.set_setting("level", json!(2000)).unwrap(); + plugin.set_setting("mode", json!("SQUARE")).unwrap(); + plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); + plugin.set_setting("min_level", json!(500)).unwrap(); + plugin.send_modulation(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.board_code == Some(500) + }); + assert!(plugin.board_mod.contains("SQUARE 500..2000")); + + plugin.output_off(); + drain_until(&mut plugin, Duration::from_secs(2), |p| { + p.board_code == Some(0) + }); + plugin.disconnect("test done"); + } + + /// min_level can never exceed the level. + #[test] + fn min_threshold_is_clamped_to_level() { + let mut plugin = StageAModulationPlugin::default(); + plugin.set_setting("level", json!(1000)).unwrap(); + plugin.set_setting("min_level", json!(3000)).unwrap(); + assert_eq!(plugin.min_level, 1000); + plugin.set_setting("level", json!(200)).unwrap(); + assert_eq!(plugin.min_level, 200, "lowering level drags min down"); + } +} diff --git a/plugins/stage-a-monitor/Cargo.toml b/plugins/stage-a-monitor/Cargo.toml deleted file mode 100644 index 61dfffd..0000000 --- a/plugins/stage-a-monitor/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "augur-plugin-stage-a-monitor" -version.workspace = true -edition.workspace = true -license.workspace = true -authors.workspace = true -description = "Stage-A commissioning monitor: live photodiode readout, calibrated optical contrast, and gated manual Teensy drive control." - -[lib] -crate-type = ["cdylib", "rlib"] - -[dependencies] -augur-plugin-api.workspace = true -serde_json.workspace = true -stage-a-io = { path = "../../stage-a-io" } diff --git a/plugins/stage-a-monitor/README.md b/plugins/stage-a-monitor/README.md deleted file mode 100644 index 8b43896..0000000 --- a/plugins/stage-a-monitor/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Stage-A Monitor - -Commissioning companion for the Stage-A camera-calibration bench: a live -view of the Teensy photodiode DAQ plus **gated** manual controller commands. - -## What it shows - -- **Photodiode waveform** — decimated calibrated trace (volts vs ms) from - the `SamplesU16` stream. -- **Live optical contrast** — `a = ln(V_max/V_min)` from dark-corrected, - clipping-guarded percentile extrema (see `stage-a-io::estimator`). An - invalid window shows *why* (clipped / no headroom / too short) instead of - a silently wrong number. -- **Stream integrity** — CRC failures, resync skips, frame-sequence gaps, - and ADC overruns. Any nonzero counter means the current point is invalid. - -## Controls (host actions on the status table) - -`Connect`, `Disconnect`, `Start acquisition`, `Stop`, and an expert -`Apply drive` modal (integer DAC codes; the optical contrast is always -measured, never assumed from the drive). Commands are actions — not -settings — so a reloaded settings file can never arm hardware. - -## Safety - -The plugin fails closed: the serial port opens only when the host reports -`LiveCapture` with `effects_allowed` (plugin ABI v5 execution context). -Replay and offline analysis can never emit a serial byte, and an existing -connection is shut down the moment effects are revoked. The firmware-side -watchdog independently drops the controller to `SAFE_IDLE` if the host -disappears. - -## Use it for (commissioning checklist) - -1. Wiring / voltage-range check at both detector loads. -2. Dark-level measurement for the estimator calibration. -3. Coherent-crosstalk test (H14): drive on, light blocked — the waveform - view and `a` readout must stay at the noise floor. -4. USB-throughput sanity (watch the integrity counters at full rate). diff --git a/plugins/stage-a-monitor/plugin.toml b/plugins/stage-a-monitor/plugin.toml deleted file mode 100644 index b82d0f5..0000000 --- a/plugins/stage-a-monitor/plugin.toml +++ /dev/null @@ -1,7 +0,0 @@ -name = "Stage-A Monitor" -version = "0.2.0" -description = "Live Teensy photodiode readout, calibrated optical contrast, and gated manual drive control for Stage-A commissioning." -domain = "stage-a" -library = "augur_plugin_stage_a_monitor" -phase = "frame_only" -min_augur_version = "1.0.0" diff --git a/plugins/stage-a-monitor/src/lib.rs b/plugins/stage-a-monitor/src/lib.rs deleted file mode 100644 index 6b3e3e9..0000000 --- a/plugins/stage-a-monitor/src/lib.rs +++ /dev/null @@ -1,831 +0,0 @@ -//! Stage-A commissioning monitor. -//! -//! Live view of the Teensy photodiode DAQ (decimated waveform, calibrated -//! optical log-contrast `a`, clipping/headroom and stream-integrity status) -//! plus **gated** manual controller commands (connect, configure, start, -//! stop) for wiring and crosstalk commissioning. -//! -//! Safety contract (Stage-A control-software spec): -//! - devices open only when `HostContext::execution()` reports -//! `LiveCapture` **and** `effects_allowed` — replay and offline analysis -//! can never touch the serial port, and a stale worker is shut down the -//! moment the context stops permitting effects; -//! - commands are host actions, never persistent settings, so a reloaded -//! settings file cannot re-arm hardware; -//! - `process_frame()` only drains the bounded I/O worker queues. - -use std::collections::BTreeMap; - -use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, - Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, - StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, -}; -use serde_json::{json, Value}; -use stage_a_io::{ - estimate_contrast, AdcCalibration, Command, ContrastEstimate, DeviceEvent, FrameType, IoWorker, - StageAClient, StreamIntegrity, WorkerOutput, WorkerRequest, -}; - -const WAVEFORM_DATASET_ID: &str = "stage-a-monitor.waveform"; -const STATUS_DATASET_ID: &str = "stage-a-monitor.status"; -const WAVEFORM_VIEW_ID: &str = "stage-a-monitor.waveform.view"; -const STATUS_VIEW_ID: &str = "stage-a-monitor.status.view"; - -const ACTION_CONNECT: &str = "stage-a-monitor.connect"; -const ACTION_DISCONNECT: &str = "stage-a-monitor.disconnect"; -const ACTION_START: &str = "stage-a-monitor.start"; -const ACTION_STOP: &str = "stage-a-monitor.stop"; -const ACTION_APPLY_DRIVE: &str = "stage-a-monitor.apply-drive"; - -/// Retained sample window for the live view + contrast estimate. -const SAMPLE_RING_CAPACITY: usize = 32_768; -/// Points published per waveform refresh (decimated). -const WAVEFORM_POINTS: usize = 1_024; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ConnectionState { - Disconnected, - Connected, - Acquiring, -} - -pub struct StageAMonitorPlugin { - enabled: bool, - // -- device -- - worker: Option, - connection: ConnectionState, - firmware: String, - next_tag: u64, - /// Tags of in-flight requests -> human-readable purpose. - in_flight: BTreeMap, - last_error: Option, - integrity: StreamIntegrity, - effects_blocked_reason: Option, - // -- settings -- - port_hint: String, - sample_rate_hz: i64, - block_samples: i64, - calibration: AdcCalibration, - // -- data -- - sample_ring: Vec, - ring_next_sample_index: u64, - sample_rate_seen_hz: u32, - contrast: Option, - contrast_error: Option, - dataset_generation: u64, - consumed_action_ids: Vec, -} - -impl Default for StageAMonitorPlugin { - fn default() -> Self { - Self { - enabled: false, - worker: None, - connection: ConnectionState::Disconnected, - firmware: String::new(), - next_tag: 1, - in_flight: BTreeMap::new(), - last_error: None, - integrity: StreamIntegrity::default(), - effects_blocked_reason: None, - port_hint: "auto".into(), - sample_rate_hz: 20_000, - block_samples: 256, - calibration: AdcCalibration::default(), - sample_ring: Vec::with_capacity(SAMPLE_RING_CAPACITY), - ring_next_sample_index: 0, - sample_rate_seen_hz: 0, - contrast: None, - contrast_error: None, - dataset_generation: 0, - consumed_action_ids: Vec::new(), - } - } -} - -impl StageAMonitorPlugin { - fn bump_generation(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - - fn connect(&mut self) { - if self.worker.is_some() { - return; - } - match open_transport(&self.port_hint) { - Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - } - Err(err) => { - self.last_error = Some(err); - } - } - self.bump_generation(); - } - - fn disconnect(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - worker.shutdown(reason); - } - self.connection = ConnectionState::Disconnected; - self.in_flight.clear(); - self.bump_generation(); - } - - fn start_acquisition(&mut self) { - self.queue_command( - "config", - Command::new("CONFIG") - .field("mode", "A1") - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", self.block_samples) - .field("raw", 1) - .field("summary", 1), - ); - self.queue_command("start", Command::new("START")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(true)); - } - } - - fn stop_acquisition(&mut self) { - self.queue_command("stop", Command::new("STOP").field("reason", "operator")); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - - fn apply_drive(&mut self, params: &Value) { - let freq_mhz = params.get("freq_mhz").and_then(Value::as_i64).unwrap_or(0); - let center_dac = params - .get("center_dac") - .and_then(Value::as_i64) - .unwrap_or(2_048); - let amplitude_dac = params - .get("amplitude_dac") - .and_then(Value::as_i64) - .unwrap_or(0); - self.queue_command( - "drive", - Command::new("CONFIG") - .field("mode", "A1") - .field("wave", "SINE") - .field("freq_mhz", freq_mhz) - .field("center_dac", center_dac) - .field("amplitude_dac", amplitude_dac) - .field("rate_hz", self.sample_rate_hz) - .field("block_samples", self.block_samples) - .field("raw", 1) - .field("summary", 1), - ); - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - if outputs.is_empty() { - return; - } - let mut changed = false; - let mut stopped: Option = None; - for output in outputs { - changed = true; - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => self.handle_reply(&purpose, &fields), - Err(err) if err.contains("unknown_config_field") => { - // Feature detection: firmware 0.2.0 has no - // waveform backend and rejects the reserved v2 - // drive fields. - self.last_error = Some(format!( - "{purpose}: firmware has no waveform backend (v1) — drive \ - control needs the mock or the future v2 firmware" - )); - } - Err(err) => { - self.last_error = Some(format!("{purpose}: {err}")); - } - } - } - WorkerOutput::Event(DeviceEvent::Data(frame)) => match frame.header.frame_type { - FrameType::SamplesU16 => { - if let Some(codes) = frame.samples() { - self.sample_rate_seen_hz = frame.header.sample_rate_hz; - self.push_samples(&codes, frame.header.first_sample_index); - } - } - FrameType::Summary | FrameType::Marker | FrameType::Control => {} - FrameType::Unknown(_) => {} - }, - WorkerOutput::Event(DeviceEvent::Async { name, fields }) => { - if name == "FAULT" { - // Firmware watchdog dropped the controller to - // SAFE_IDLE — reflect it instead of showing a stale - // "acquiring" state. - if self.connection == ConnectionState::Acquiring { - self.connection = ConnectionState::Connected; - } - self.last_error = Some(format!( - "controller fault: {} — dropped to SAFE_IDLE", - fields.get("code").map(String::as_str).unwrap_or("unknown") - )); - if let Some(worker) = &self.worker { - let _ = worker.try_send(WorkerRequest::SetPinging(false)); - } - } - } - WorkerOutput::Integrity(integrity) => { - self.integrity = integrity; - } - WorkerOutput::Stopped { reason } => { - stopped = Some(reason); - } - } - } - if let Some(reason) = stopped { - self.worker = None; - self.connection = ConnectionState::Disconnected; - self.last_error = Some(format!("device connection ended: {reason}")); - } - if changed { - self.refresh_contrast(); - self.bump_generation(); - } - } - - fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { - match purpose { - "hello" => { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - self.connection = ConnectionState::Connected; - } - "start" => { - self.connection = ConnectionState::Acquiring; - } - "stop" => { - self.connection = ConnectionState::Connected; - } - _ => {} - } - } - - fn push_samples(&mut self, codes: &[u16], first_sample_index: u64) { - self.ring_next_sample_index = first_sample_index + codes.len() as u64; - self.sample_ring.extend_from_slice(codes); - let len = self.sample_ring.len(); - if len > SAMPLE_RING_CAPACITY { - self.sample_ring.drain(..len - SAMPLE_RING_CAPACITY); - } - } - - fn refresh_contrast(&mut self) { - if self.sample_ring.len() < stage_a_io::estimator::MIN_SAMPLES { - return; - } - match estimate_contrast(&self.sample_ring, &self.calibration) { - Ok(estimate) => { - self.contrast = Some(estimate); - self.contrast_error = None; - } - Err(err) => { - self.contrast = None; - self.contrast_error = Some(err.to_string()); - } - } - } - - fn waveform_dataset(&self) -> Series1dV1 { - let rate = if self.sample_rate_seen_hz > 0 { - f64::from(self.sample_rate_seen_hz) - } else { - self.sample_rate_hz as f64 - }; - let n = self.sample_ring.len(); - let stride = (n / WAVEFORM_POINTS).max(1); - let first_index = self.ring_next_sample_index.saturating_sub(n as u64); - let points: Vec = self - .sample_ring - .iter() - .enumerate() - .step_by(stride) - .map(|(i, &code)| Series1dPoint { - x: (first_index + i as u64) as f64 / rate * 1_000.0, - y: self.calibration.code_to_volts(code), - }) - .collect(); - Series1dV1 { - x_label: "time [ms]".into(), - y_label: "photodiode [V]".into(), - lines: vec![Series1dLine { - name: "photodiode".into(), - points, - }], - } - } - - fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.connection) { - (Some(reason), _) => format!("locked ({reason})"), - (None, ConnectionState::Disconnected) => "disconnected".into(), - (None, ConnectionState::Connected) => "connected".into(), - (None, ConnectionState::Acquiring) => "acquiring".into(), - }; - let (a_text, clip_text) = match (&self.contrast, &self.contrast_error) { - (Some(estimate), _) => ( - format!("{:.4}", estimate.a), - format!( - "{:.2}% low / {:.2}% high", - estimate.low_clip_fraction * 100.0, - estimate.high_clip_fraction * 100.0 - ), - ), - (None, Some(err)) => ("invalid".into(), err.clone()), - (None, None) => ("—".into(), "—".into()), - }; - let integrity = if self.integrity.is_clean() { - "clean".to_owned() - } else { - format!( - "crc={} gaps={} skipped={} overruns={}", - self.integrity.crc_failures, - self.integrity.sequence_gaps, - self.integrity.skipped_bytes, - self.integrity.dropped_samples - ) - }; - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", state), - text_column("firmware", self.firmware.clone()), - text_column("a", a_text), - text_column("clipping", clip_text), - text_column("integrity", integrity), - text_column("error", self.last_error.clone().unwrap_or_default()), - ], - } - } - - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("firmware", "Firmware"), - column("a", "a = ln(Vmax/Vmin)"), - column("clipping", "Clipping"), - column("integrity", "Stream integrity"), - column("error", "Last error"), - ], - ..TableSchema::default() - } - } - - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec<(String, Value)> { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-monitor.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push((request.action_id, request.params)); - } - consumed - } -} - -fn open_transport(port_hint: &str) -> Result, String> { - let path = resolve_port(port_hint)?; - let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} - -fn resolve_port(port_hint: &str) -> Result { - if port_hint != "auto" { - return Ok(port_hint.to_owned()); - } - let ports = serial_ports(); - ports - .into_iter() - .next() - .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned()) -} - -fn serial_ports() -> Vec { - serialport_names() - .into_iter() - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) - .collect() -} - -fn serialport_names() -> Vec { - stage_a_io::transport::available_port_names() -} - -impl Plugin for StageAMonitorPlugin { - fn name(&self) -> &'static str { - "Stage-A Monitor" - } - - fn description(&self) -> &'static str { - "Live Teensy photodiode readout with calibrated optical contrast and gated manual drive control (commissioning)." - } - - fn enabled(&self) -> bool { - self.enabled - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.disconnect("plugin disabled"); - } - } - - fn reset(&mut self) { - self.sample_ring.clear(); - self.contrast = None; - self.contrast_error = None; - self.bump_generation(); - } - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - // Fail closed: any pass without live-capture effects tears the - // connection down and refuses commands. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.worker.is_some() { - self.disconnect("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for (action_id, params) in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect("operator"), - ACTION_START => self.start_acquisition(), - ACTION_STOP => self.stop_acquisition(), - ACTION_APPLY_DRIVE => self.apply_drive(¶ms), - _ => {} - } - } - - self.drain_worker(); - } - - fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["auto".to_owned()]; - port_variants.extend(serial_ports()); - let port_default = port_variants - .iter() - .position(|p| *p == self.port_hint) - .unwrap_or(0); - SettingsSchema { - sections: vec![SettingsSection { - label: "Device".into(), - description: Some( - "Serial DAQ configuration. Connect/start/stop are actions on the status \ - table, never settings — a reloaded settings file can't arm hardware." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Serial port".into(), - tooltip: Some("Teensy USB serial device (auto = first usbmodem)".into()), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, - }, - }, - SettingItem { - key: "sample_rate_hz".into(), - label: "ADC sample rate".into(), - tooltip: Some("Commanded photodiode sample rate".into()), - kind: SettingKind::I64Slider { - min: 1_000, - max: 100_000, - default: self.sample_rate_hz, - suffix: Some(" Hz".into()), - }, - }, - SettingItem { - key: "dark_millivolts".into(), - label: "Dark level".into(), - tooltip: Some( - "Light-blocked photodiode level; a is computed from dark-corrected \ - voltages" - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 3_300.0, - speed: 1.0, - default: self.calibration.dark_volts * 1_000.0, - }, - }, - ], - }], - } - } - - fn get_setting(&self, key: &str) -> Option { - match key { - "port" => Some(json!(self.port_hint)), - "sample_rate_hz" => Some(json!(self.sample_rate_hz)), - "dark_millivolts" => Some(json!(self.calibration.dark_volts * 1_000.0)), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); - Ok(()) - } - "sample_rate_hz" => { - self.sample_rate_hz = value - .as_i64() - .ok_or("sample_rate_hz must be an integer")? - .clamp(1_000, 100_000); - Ok(()) - } - "dark_millivolts" => { - let mv = value.as_f64().ok_or("dark_millivolts must be a number")?; - self.calibration.dark_volts = (mv / 1_000.0).clamp(0.0, 3.3); - Ok(()) - } - _ => Err(format!("unknown setting: {key}")), - } - } - - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - entries.push(StatusEntry::Text(match self.connection { - ConnectionState::Disconnected => "Teensy: disconnected".into(), - ConnectionState::Connected => format!("Teensy: connected ({})", self.firmware), - ConnectionState::Acquiring => format!( - "Teensy: acquiring at {} S/s", - if self.sample_rate_seen_hz > 0 { - self.sample_rate_seen_hz as i64 - } else { - self.sample_rate_hz - } - ), - })); - if let Some(estimate) = &self.contrast { - entries.push(StatusEntry::Text(format!("a = {:.4}", estimate.a))); - } - entries - } - - fn host_views(&self) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![ - HostDatasetDescriptor { - id: WAVEFORM_DATASET_ID.into(), - title: "Photodiode waveform".into(), - kind: HostDatasetKind::Series1dV1, - empty_message: "No photodiode samples yet — connect and start.".into(), - display: None, - relations: Vec::new(), - }, - HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "Stage-A monitor status".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Monitor idle.".into(), - display: None, - relations: Vec::new(), - }, - ], - views: vec![ - HostViewDescriptor { - id: WAVEFORM_VIEW_ID.into(), - title: "Photodiode".into(), - dataset_id: WAVEFORM_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::LineSeriesWindow, - }, - HostViewDescriptor { - id: STATUS_VIEW_ID.into(), - title: "Monitor status".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - ], - actions: vec![ - HostActionDescriptor { - id: ACTION_CONNECT.into(), - title: "Connect".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_DISCONNECT.into(), - title: "Disconnect".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_START.into(), - title: "Start acquisition".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_STOP.into(), - title: "Stop".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }, - HostActionDescriptor { - id: ACTION_APPLY_DRIVE.into(), - title: "Apply drive (expert)".into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: serde_json::to_value(SettingsSchema { - sections: vec![SettingsSection { - label: "Drive".into(), - description: Some( - "Integer DAC drive codes — the optical contrast is measured \ - from the photodiode, never assumed from these values." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "freq_mhz".into(), - label: "Frequency".into(), - tooltip: Some("Drive frequency in millihertz".into()), - kind: SettingKind::I64Drag { - min: 0, - max: 200_000_000, - default: 1_000_000, - }, - }, - SettingItem { - key: "center_dac".into(), - label: "Center DAC code".into(), - tooltip: None, - kind: SettingKind::I64Drag { - min: 0, - max: 4_095, - default: 2_048, - }, - }, - SettingItem { - key: "amplitude_dac".into(), - label: "Amplitude DAC code".into(), - tooltip: None, - kind: SettingKind::I64Drag { - min: 0, - max: 2_047, - default: 0, - }, - }, - ], - }], - }) - .ok(), - }, - ], - } - } - - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - WAVEFORM_DATASET_ID => serde_json::to_vec(&self.waveform_dataset()).ok(), - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, - } - } - - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - WAVEFORM_DATASET_ID | STATUS_DATASET_ID => self.dataset_generation.max(1), - _ => 0, - } - } -} - -impl Drop for StageAMonitorPlugin { - fn drop(&mut self) { - self.disconnect("plugin destroyed"); - } -} - -export_plugin!(StageAMonitorPlugin); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn waveform_dataset_decimates_and_calibrates() { - let mut plugin = StageAMonitorPlugin::default(); - plugin.sample_rate_seen_hz = 20_000; - plugin.push_samples(&vec![2_048_u16; 8_192], 0); - let dataset = plugin.waveform_dataset(); - assert_eq!(dataset.lines.len(), 1); - assert!(dataset.lines[0].points.len() <= WAVEFORM_POINTS + 1); - let volts = dataset.lines[0].points[0].y; - assert!((volts - 2_048.0 * 3.3 / 4_095.0).abs() < 1e-9); - } - - #[test] - fn sample_ring_is_bounded() { - let mut plugin = StageAMonitorPlugin::default(); - plugin.push_samples(&vec![1_u16; SAMPLE_RING_CAPACITY], 0); - plugin.push_samples(&vec![2_u16; 4_096], SAMPLE_RING_CAPACITY as u64); - assert_eq!(plugin.sample_ring.len(), SAMPLE_RING_CAPACITY); - assert_eq!(*plugin.sample_ring.last().unwrap(), 2); - } - - #[test] - fn status_dataset_matches_its_schema() { - let plugin = StageAMonitorPlugin::default(); - let dataset = plugin.status_dataset(); - let schema = plugin.status_schema(); - assert_eq!(dataset.columns.len(), schema.columns.len()); - for (data, column) in dataset.columns.iter().zip(&schema.columns) { - assert_eq!(data.column_id, column.id); - assert_eq!(data.len(), 1); - } - } -} diff --git a/plugins/stage-a-a1/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml similarity index 51% rename from plugins/stage-a-a1/Cargo.toml rename to plugins/stage-a-photodiode/Cargo.toml index fa4841d..b426623 100644 --- a/plugins/stage-a-a1/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "augur-plugin-stage-a-a1" +name = "augur-plugin-stage-a-photodiode" version.workspace = true edition.workspace = true license.workspace = true authors.workspace = true -description = "Stage-A A1 event-native Bode calibration: minimum-depth a_min(f) sweep with phase-locked detection." +description = "Stage-A photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." [lib] crate-type = ["cdylib", "rlib"] @@ -12,4 +12,4 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true serde_json.workspace = true -stage-a-io = { path = "../../stage-a-io" } +serialport.workspace = true diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md new file mode 100644 index 0000000..5f59e82 --- /dev/null +++ b/plugins/stage-a-photodiode/README.md @@ -0,0 +1,25 @@ +# Stage-A Photodiode + +Live readout of the photodiode on **board SMA5 → Teensy pin 18 / analog input A4**, from the +free-running ASCII stream the `stage-a-controller` firmware (0.3.0+) emits on its **second** USB +serial port (`PD code=… n=… t_ms=…` at 50 Hz). The port carries no commands, so this plugin is +read-only by construction; the command port belongs to `stage-a-modulation`. + +## Modes + +- **RAW** — shows the ADC code and its voltage, `V = code · 3.3 / 4095`. +- **EXCITATION** — the photodiode sits in the excitation path behind the PBS and measures the + light *removed* from the beam: `I_pd = I_tot − I_exc`. Given the user-set reference **I_tot** + (in photodiode volts — the reading with the full beam on the diode), the plugin shows + `I_exc = I_tot − I_pd`. + +## Views + +- a live rolling chart (window length settable, 1–120 s) of the value in the selected mode; +- a compact status table with the newest code/value and Connect/Disconnect actions. + +## Ports + +Select the Teensy *stream* port (the second `usbmodem` port). Picking the command port by mistake +is harmless: its binary frames simply parse to nothing (no values appear) — switch to the other +port. `mock` generates a synthetic slow sine for hardware-free testing. diff --git a/plugins/stage-a-photodiode/plugin.toml b/plugins/stage-a-photodiode/plugin.toml new file mode 100644 index 0000000..702a218 --- /dev/null +++ b/plugins/stage-a-photodiode/plugin.toml @@ -0,0 +1,7 @@ +name = "Stage-A Photodiode" +version = "0.3.0" +description = "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." +domain = "stage-a" +library = "augur_plugin_stage_a_photodiode" +phase = "frame_only" +min_augur_version = "1.0.0" diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs new file mode 100644 index 0000000..8a6f3a7 --- /dev/null +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -0,0 +1,774 @@ +//! Stage-A photodiode readout. +//! +//! Reads the free-running ASCII stream the `stage-a-controller` firmware +//! (0.3.0+, `USB_DUAL_SERIAL`) emits on its **second** USB serial port: +//! one `PD code= n= t_ms=` line every 20 ms from the +//! photodiode on board SMA5 → Teensy pin 18 / A4. The port carries no +//! commands, so opening it is side-effect free; the command port is owned by +//! `stage-a-modulation`. +//! +//! Two display modes: +//! - **RAW**: the ADC code and its voltage (`V = code · 3.3 / 4095`); +//! - **EXCITATION**: the photodiode sits behind the PBS in the excitation +//! path and sees the light removed from the beam, `I_pd = I_tot − I_exc`. +//! Given the user-set reference `I_tot` (in photodiode volts), the plugin +//! shows `I_exc = I_tot − V_pd`. + +use std::collections::VecDeque; +use std::io::Read; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, + HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, + HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, + Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, + StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, + TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, +}; +use serde_json::{json, Value}; + +const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; +const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; +const STATUS_DATASET_ID: &str = "stage-a-photodiode.status"; +const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; + +const ACTION_CONNECT: &str = "stage-a-photodiode.connect"; +const ACTION_DISCONNECT: &str = "stage-a-photodiode.disconnect"; + +const ADC_FULL_SCALE_VOLTS: f64 = 3.3; +const ADC_MAX_CODE: f64 = 4_095.0; +/// Ring capacity: > 2.5 minutes at the firmware's 50 lines/s. +const RING_CAPACITY: usize = 8_192; + +fn code_to_volts(code: f64) -> f64 { + code * ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Raw, + Excitation, +} + +impl Mode { + const VARIANTS: [Mode; 2] = [Mode::Raw, Mode::Excitation]; + + fn name(self) -> &'static str { + match self { + Self::Raw => "RAW", + Self::Excitation => "EXCITATION", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|m| m.name() == name) + } +} + +#[derive(Debug, Clone, Copy)] +struct PdSample { + t_ms: u64, + code: f64, +} + +/// Parses one firmware stream line: `PD code= n= t_ms=`. +fn parse_pd_line(line: &str) -> Option { + let rest = line.trim().strip_prefix("PD ")?; + let mut code = None; + let mut t_ms = None; + for token in rest.split_ascii_whitespace() { + let (key, value) = token.split_once('=')?; + match key { + "code" => code = value.parse::().ok(), + "t_ms" => t_ms = value.parse::().ok(), + "n" => {} + _ => return None, + } + } + Some(PdSample { + t_ms: t_ms?, + code: code?.clamp(0.0, ADC_MAX_CODE), + }) +} + +#[derive(Default)] +struct SharedState { + samples: VecDeque, + latest: Option, + error: Option, +} + +impl SharedState { + fn push(&mut self, sample: PdSample) { + self.latest = Some(sample); + self.samples.push_back(sample); + while self.samples.len() > RING_CAPACITY { + self.samples.pop_front(); + } + } +} + +/// Background reader owning the stream port (or the mock generator). +struct Reader { + stop: Arc, + join: Option>, +} + +impl Reader { + fn spawn_serial( + path: String, + shared: Arc>, + generation: Arc, + ) -> Result { + let port = serialport::new(&path, 115_200) + .timeout(Duration::from_millis(50)) + .open() + .map_err(|err| format!("open {path}: {err}"))?; + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let join = std::thread::Builder::new() + .name("stage-a-photodiode".into()) + .spawn(move || read_lines(port, &shared, &generation, &thread_stop)) + .expect("spawning the photodiode reader thread must succeed"); + Ok(Self { + stop, + join: Some(join), + }) + } + + /// Hardware-free source: synthesizes a slow sine around 1 V at 50 Hz. + fn spawn_mock(shared: Arc>, generation: Arc) -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let thread_stop = Arc::clone(&stop); + let join = std::thread::Builder::new() + .name("stage-a-photodiode-mock".into()) + .spawn(move || { + let start = Instant::now(); + while !thread_stop.load(Ordering::Relaxed) { + let t = start.elapsed().as_secs_f64(); + let volts = 1.0 + 0.5 * (2.0 * std::f64::consts::PI * 0.2 * t).sin(); + let sample = PdSample { + t_ms: (t * 1_000.0) as u64, + code: volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS, + }; + if let Ok(mut state) = shared.lock() { + state.push(sample); + } + generation.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(20)); + } + }) + .expect("spawning the mock photodiode thread must succeed"); + Self { + stop, + join: Some(join), + } + } +} + +impl Drop for Reader { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +fn read_lines( + mut port: Box, + shared: &Mutex, + generation: &AtomicU64, + stop: &AtomicBool, +) { + let mut line_buffer: Vec = Vec::with_capacity(256); + let mut buf = [0_u8; 512]; + while !stop.load(Ordering::Relaxed) { + let read = match port.read(&mut buf) { + Ok(0) => continue, + Ok(read) => read, + Err(err) if err.kind() == std::io::ErrorKind::TimedOut => continue, + Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, + Err(err) => { + if let Ok(mut state) = shared.lock() { + state.error = Some(format!("stream read failed: {err}")); + } + generation.fetch_add(1, Ordering::Relaxed); + return; + } + }; + line_buffer.extend_from_slice(&buf[..read]); + // Never let garbage (e.g. the wrong, binary port) grow the buffer. + if line_buffer.len() > 4_096 { + line_buffer.clear(); + } + while let Some(pos) = line_buffer.iter().position(|&b| b == b'\n') { + let line: Vec = line_buffer.drain(..=pos).collect(); + let Ok(text) = std::str::from_utf8(&line) else { + continue; + }; + if let Some(sample) = parse_pd_line(text) { + if let Ok(mut state) = shared.lock() { + state.push(sample); + } + generation.fetch_add(1, Ordering::Relaxed); + } + } + } +} + +pub struct StageAPhotodiodePlugin { + enabled: bool, + reader: Option, + shared: Arc>, + generation: Arc, + effects_blocked_reason: Option, + last_error: Option, + // -- settings -- + port_hint: String, + mode: Mode, + reference_volts: f64, + window_s: f64, + consumed_action_ids: Vec, +} + +impl Default for StageAPhotodiodePlugin { + fn default() -> Self { + Self { + enabled: false, + reader: None, + shared: Arc::new(Mutex::new(SharedState::default())), + generation: Arc::new(AtomicU64::new(1)), + effects_blocked_reason: None, + last_error: None, + port_hint: "mock".into(), + mode: Mode::Raw, + reference_volts: 3.3, + window_s: 10.0, + consumed_action_ids: Vec::new(), + } + } +} + +impl StageAPhotodiodePlugin { + fn connected(&self) -> bool { + self.reader.is_some() + } + + fn connect(&mut self) { + if self.reader.is_some() { + return; + } + if let Ok(mut state) = self.shared.lock() { + *state = SharedState::default(); + } + self.last_error = None; + if self.port_hint == "mock" { + self.reader = Some(Reader::spawn_mock( + Arc::clone(&self.shared), + Arc::clone(&self.generation), + )); + return; + } + let path = if self.port_hint == "auto" { + match serial_ports().into_iter().next() { + Some(path) => path, + None => { + self.last_error = + Some("no USB serial device found (looked for usbmodem/ttyACM)".into()); + return; + } + } + } else { + self.port_hint.clone() + }; + match Reader::spawn_serial(path, Arc::clone(&self.shared), Arc::clone(&self.generation)) { + Ok(reader) => self.reader = Some(reader), + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + + fn disconnect(&mut self) { + self.reader = None; // Drop joins the thread. + self.generation.fetch_add(1, Ordering::Relaxed); + } + + /// Value shown for one sample under the current mode, in volts. + fn display_volts(&self, code: f64) -> f64 { + match self.mode { + Mode::Raw => code_to_volts(code), + Mode::Excitation => self.reference_volts - code_to_volts(code), + } + } + + fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { + let Ok(Some(queue)) = + context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) + else { + return Vec::new(); + }; + let mut consumed = Vec::new(); + for request in queue.requests { + if self.consumed_action_ids.contains(&request.request_id) { + continue; + } + if !request.action_id.starts_with("stage-a-photodiode.") { + continue; + } + self.consumed_action_ids.push(request.request_id); + if self.consumed_action_ids.len() > 256 { + self.consumed_action_ids.remove(0); + } + consumed.push(request.action_id); + } + consumed + } + + fn series_dataset(&self) -> Series1dV1 { + let (points, y_label) = match self.shared.lock() { + Ok(state) => { + let latest_ms = state.latest.map_or(0, |s| s.t_ms); + let window_ms = (self.window_s.max(0.5) * 1_000.0) as u64; + let cutoff = latest_ms.saturating_sub(window_ms); + let points: Vec = state + .samples + .iter() + .filter(|s| s.t_ms >= cutoff) + .map(|s| Series1dPoint { + x: (s.t_ms as f64 - latest_ms as f64) / 1_000.0, + y: self.display_volts(s.code), + }) + .collect(); + let label = match self.mode { + Mode::Raw => "photodiode [V]", + Mode::Excitation => "excitation I_tot − I_pd [V]", + }; + (points, label) + } + Err(_) => (Vec::new(), "photodiode [V]"), + }; + Series1dV1 { + x_label: "time before now [s]".into(), + y_label: y_label.into(), + lines: vec![Series1dLine { + name: match self.mode { + Mode::Raw => "photodiode".into(), + Mode::Excitation => "excitation".into(), + }, + points, + }], + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let (latest, stream_error) = match self.shared.lock() { + Ok(state) => (state.latest, state.error.clone()), + Err(_) => (None, None), + }; + let state = match (&self.effects_blocked_reason, self.connected()) { + (Some(reason), _) => format!("locked ({reason})"), + (None, false) => "disconnected".into(), + (None, true) => format!("reading ({})", self.port_hint), + }; + let (code_text, value_text) = match latest { + Some(sample) => ( + format!("{:.1}", sample.code), + format!("{:.4} V", self.display_volts(sample.code)), + ), + None => ("—".into(), "—".into()), + }; + let error = stream_error + .or_else(|| self.last_error.clone()) + .unwrap_or_default(); + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", state), + text_column("mode", self.mode.name().to_owned()), + text_column("code", code_text), + text_column("value", value_text), + text_column("error", error), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("mode", "Mode"), + column("code", "ADC code"), + column("value", "Value"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } +} + +fn serial_ports() -> Vec { + serialport::available_ports() + .map(|ports| { + ports + .into_iter() + .map(|p| p.port_name) + .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + .collect() + }) + .unwrap_or_default() +} + +impl Plugin for StageAPhotodiodePlugin { + fn name(&self) -> &'static str { + "Stage-A Photodiode" + } + + fn description(&self) -> &'static str { + "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot − I_pd with a user-set reference." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.disconnect(); + } + } + + fn reset(&mut self) { + if let Ok(mut state) = self.shared.lock() { + state.samples.clear(); + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // The stream port is read-only, but device access still follows the + // same fail-closed gate as every stage-a plugin. + let execution = context.execution(); + if !execution.hardware_effects_allowed() { + self.effects_blocked_reason = Some(format!( + "hardware effects not allowed in {:?}", + execution.mode + )); + if self.reader.is_some() { + self.disconnect(); + } + return; + } + self.effects_blocked_reason = None; + + for action_id in self.consume_actions(context) { + match action_id.as_str() { + ACTION_CONNECT => self.connect(), + ACTION_DISCONNECT => self.disconnect(), + _ => {} + } + } + } + + fn settings_schema(&self) -> SettingsSchema { + let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; + port_variants.extend(serial_ports()); + let port_default = port_variants + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + let mode_variants: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let mode_default = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + SettingsSchema { + sections: vec![SettingsSection { + label: "Photodiode readout".into(), + description: Some( + "Reads the free-running PD stream on the Teensy's SECOND serial port. \ + EXCITATION shows I_exc = I_tot − I_pd: the diode sits behind the PBS and \ + sees the light removed from the excitation beam." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "Teensy stream port (the SECOND usbmodem port); mock = synthetic \ + data, auto = first device" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some( + "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd".into(), + ), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, + }, + SettingItem { + key: "reference_volts".into(), + label: "Reference I_tot".into(), + tooltip: Some( + "Total power reference for EXCITATION mode, in photodiode volts: \ + the PD reading with the full beam diverted into the diode" + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: ADC_FULL_SCALE_VOLTS, + speed: 0.01, + default: self.reference_volts, + }, + }, + SettingItem { + key: "window_s".into(), + label: "Chart window".into(), + tooltip: Some("Seconds of history shown in the live chart".into()), + kind: SettingKind::F64Drag { + min: 1.0, + max: 120.0, + speed: 1.0, + default: self.window_s, + }, + }, + ], + }], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "port" => Some(json!(self.port_hint)), + "mode" => Some(json!(self.mode.name())), + "reference_volts" => Some(json!(self.reference_volts)), + "window_s" => Some(json!(self.window_s)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "port" => { + self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + Ok(()) + } + "mode" => { + let name = value.as_str().ok_or("mode must be a string")?; + self.mode = Mode::from_name(name) + .ok_or_else(|| format!("unknown mode: {name} (RAW/EXCITATION)"))?; + Ok(()) + } + "reference_volts" => { + let volts = value.as_f64().ok_or("reference_volts must be a number")?; + self.reference_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); + Ok(()) + } + "window_s" => { + let seconds = value.as_f64().ok_or("window_s must be a number")?; + self.window_s = seconds.clamp(1.0, 120.0); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + if let Some(reason) = &self.effects_blocked_reason { + entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); + } + let (latest, stream_error) = match self.shared.lock() { + Ok(state) => (state.latest, state.error.clone()), + Err(_) => (None, None), + }; + entries.push(StatusEntry::Text(if self.connected() { + format!("Photodiode: reading ({})", self.port_hint) + } else { + "Photodiode: disconnected".into() + })); + if let Some(sample) = latest { + match self.mode { + Mode::Raw => entries.push(StatusEntry::Text(format!( + "PD: code={:.1} ({:.4} V)", + sample.code, + code_to_volts(sample.code) + ))), + Mode::Excitation => entries.push(StatusEntry::Text(format!( + "Excitation: {:.4} V (I_tot={:.3} V, PD={:.4} V)", + self.display_volts(sample.code), + self.reference_volts, + code_to_volts(sample.code) + ))), + } + } + if let Some(error) = stream_error.or_else(|| self.last_error.clone()) { + entries.push(StatusEntry::Text(format!("Error: {error}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + let action = |id: &str, title: &str| HostActionDescriptor { + id: id.into(), + title: title.into(), + scope: HostActionScope::Dataset { + dataset_id: STATUS_DATASET_ID.into(), + }, + param_schema: None, + }; + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: SERIES_DATASET_ID.into(), + title: "Photodiode trace".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "No photodiode samples yet — connect the stream port.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Photodiode readout".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Photodiode readout idle.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: SERIES_VIEW_ID.into(), + title: "Photodiode".into(), + dataset_id: SERIES_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Photodiode readout".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + ], + actions: vec![ + action(ACTION_CONNECT, "Connect"), + action(ACTION_DISCONNECT, "Disconnect"), + ], + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + SERIES_DATASET_ID => serde_json::to_vec(&self.series_dataset()).ok(), + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + SERIES_DATASET_ID | STATUS_DATASET_ID => self.generation.load(Ordering::Relaxed).max(1), + _ => 0, + } + } +} + +export_plugin!(StageAPhotodiodePlugin); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_firmware_stream_lines() { + let sample = parse_pd_line("PD code=1042.3 n=16 t_ms=123456\n").expect("valid line"); + assert!((sample.code - 1042.3).abs() < 1e-9); + assert_eq!(sample.t_ms, 123_456); + + assert!(parse_pd_line("garbage").is_none()); + assert!(parse_pd_line("PD code=abc n=16 t_ms=1").is_none()); + assert!(parse_pd_line("PD code=10 n=16").is_none(), "t_ms required"); + // Codes are clamped into the 12-bit range. + let clamped = parse_pd_line("PD code=9999 n=1 t_ms=5").expect("parses"); + assert_eq!(clamped.code, ADC_MAX_CODE); + } + + #[test] + fn excitation_mode_inverts_against_the_reference() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("mode", json!("EXCITATION")).unwrap(); + plugin.set_setting("reference_volts", json!(2.0)).unwrap(); + // I_pd = 0.5 V → I_exc = I_tot − I_pd = 1.5 V. + let code = 0.5 * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS; + assert!((plugin.display_volts(code) - 1.5).abs() < 1e-9); + // RAW mode shows the measured voltage itself. + plugin.set_setting("mode", json!("RAW")).unwrap(); + assert!((plugin.display_volts(code) - 0.5).abs() < 1e-9); + } + + #[test] + fn mock_reader_fills_the_ring_and_series() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.connect(); + let deadline = Instant::now() + Duration::from_secs(2); + loop { + let count = plugin.shared.lock().unwrap().samples.len(); + if count >= 5 { + break; + } + assert!(Instant::now() < deadline, "mock reader produced no data"); + std::thread::sleep(Duration::from_millis(5)); + } + let series = plugin.series_dataset(); + assert!(!series.lines[0].points.is_empty()); + let generation = plugin.generation.load(Ordering::Relaxed); + assert!(generation > 1); + plugin.disconnect(); + } + + #[test] + fn ring_is_bounded() { + let mut state = SharedState::default(); + for i in 0..(RING_CAPACITY + 100) { + state.push(PdSample { + t_ms: i as u64, + code: 1.0, + }); + } + assert_eq!(state.samples.len(), RING_CAPACITY); + assert_eq!(state.latest.unwrap().t_ms, (RING_CAPACITY + 99) as u64); + } +} diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index 2a21ff7..2a9eabd 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -1,7 +1,8 @@ //! # stage-a-io //! -//! Shared research-owned I/O library for the Stage-A camera-calibration -//! plugins (`stage-a-monitor`, `stage-a-a1`, `stage-a-a2`, `stage-a-a3`). +//! Shared research-owned I/O library for the Stage-A bench plugins +//! (currently `stage-a-modulation`; the future A1–A3 experiment plugins +//! build on it too — see ADR 006). //! //! Scope, per the Stage-A control-software specification: //! - the v1 ASCII command grammar and PDA1 binary frame format (wire- diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs index 803d919..203733c 100644 --- a/stage-a-io/src/mock.rs +++ b/stage-a-io/src/mock.rs @@ -1,12 +1,13 @@ //! Mock Stage-A controller for tests and hardware-free plugin development. //! -//! Mirrors firmware 0.2.0 (`stage-a-controller/src/main.cpp`) faithfully: -//! the same verbs (`HELLO`, `STATUS`, `CONFIG`, `START`, `STOP`, `PING`), -//! the same state machine (`SAFE_IDLE` → `CONFIGURED` → `RUNNING`), the -//! same error codes/details (`PROTOCOL`, `RANGE`, `STATE`, `SYNTAX`, +//! Mirrors firmware 0.3.0 (`stage-a-controller/src/main.cpp`) faithfully: +//! the same verbs (`HELLO`, `STATUS`, `CONFIG`, `START`, `STOP`, `PING`, +//! `MOD`), the same state machine (`SAFE_IDLE` → `CONFIGURED` → `RUNNING`), +//! the same error codes/details (`PROTOCOL`, `RANGE`, `STATE`, `SYNTAX`, //! `VERB`), the same single-entry idempotent reply cache, and rejection of //! unknown `CONFIG` fields — which is the host's feature-detection -//! mechanism, so it must never be papered over here. +//! mechanism, so it must never be papered over here. `MOD` is set-and-hold +//! exactly like the firmware: `STOP` does not touch the modulation state. //! //! [`MockController::with_waveform_extension`] additionally models the //! *proposed* v2 waveform firmware (`stage-a-controller/docs/features/` @@ -24,6 +25,9 @@ pub const MOCK_MAX_RATE_HZ: u32 = 100_000; pub const MOCK_MAX_BLOCK_SAMPLES: u32 = 256; /// Proposed v2 waveform ceiling (matches the drive UI bound: 200 kHz). pub const MOCK_MAX_FREQ_MHZ: u32 = 200_000_000; +/// Firmware 0.3.0 `MOD` frequency window (`board_config.h`). +pub const MOCK_MOD_MIN_FREQ_MHZ: u32 = 10; +pub const MOCK_MOD_MAX_FREQ_MHZ: u32 = 2_000_000; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum MockState { @@ -111,6 +115,12 @@ pub struct MockController { out_sequence: u32, line_buffer: Vec, sample_index: u64, + // Firmware 0.3.0 MOD state (set-and-hold, independent of acquisition). + mod_wave: &'static str, + mod_level: u32, + mod_min: u32, + mod_freq_mhz: u32, + mod_code: u32, /// Synthetic optics for [`MockController::emit_configured_block`]: /// photodiode code = dark + span * sin²(π/2 · drive/4095). pub synth_dark_code: f64, @@ -134,6 +144,11 @@ impl MockController { out_sequence: 0, line_buffer: Vec::new(), sample_index: 0, + mod_wave: "OFF", + mod_level: 0, + mod_min: 0, + mod_freq_mhz: 0, + mod_code: 0, synth_dark_code: 40.0, synth_span_codes: 3_800.0, synth_center: 2_048.0, @@ -273,19 +288,20 @@ impl MockController { return format!("-{sequence} ERR code=PROTOCOL detail=requires_v1"); } let capabilities = if self.waveform_extension { - " capabilities=A1,A2,A3,WAVE" + " capabilities=MOD,PDSTREAM,WAVE" } else { - "" + " capabilities=MOD,PDSTREAM" }; format!( - "+{sequence} OK protocol=1 firmware=0.2.0-mock board=MOCK adc_bits=12 \ + "+{sequence} OK protocol=1 firmware=0.3.0-mock board=MOCK adc_bits=12 \ max_rate_hz={MOCK_MAX_RATE_HZ} dac=AD5628 dac_bus=SPI1 dac_cs=29 \ - dac_channel=1.4 dac_address=3{capabilities}" + dac_channel=1.4 dac_address=3 pd_pin=A4{capabilities}" ) } "STATUS" => format!( "+{sequence} OK state={} mode={} rate_hz={} block_samples={} raw={} summary={} \ - sample_index={} dropped=0 marker_drops=0 dac=1.4/3 code=0", + sample_index={} dropped=0 marker_drops=0 dac=1.4/3 code={} mod_wave={} \ + mod_level={} mod_min={} mod_freq_mhz={}", self.state.name(), self.config.mode, self.config.rate_hz, @@ -293,8 +309,14 @@ impl MockController { u8::from(self.config.raw), u8::from(self.config.summary), self.sample_index, + self.mod_code, + self.mod_wave, + self.mod_level, + self.mod_min, + self.mod_freq_mhz, ), "CONFIG" => self.execute_config(fields, sequence), + "MOD" => self.execute_mod(fields, sequence), "START" => { if self.state != MockState::Configured { return format!("-{sequence} ERR code=STATE detail=configure_before_start"); @@ -403,6 +425,86 @@ impl MockController { ) } + /// Firmware 0.3.0 `MOD` handler: same field grammar, validation order, + /// error details, and reply shape as `main.cpp`. + fn execute_mod(&mut self, fields: &[(String, String)], sequence: u32) -> String { + let err = |code: &str, detail: &str| format!("-{sequence} ERR code={code} detail={detail}"); + let mut wave: Option<&'static str> = None; + let mut level = 0_u32; + let mut saw_level = false; + let mut min_level = 0_u32; + let mut freq_mhz = 0_u32; + let mut saw_freq = false; + for (key, value) in fields { + match key.as_str() { + "wave" => { + wave = Some(match value.as_str() { + "OFF" => "OFF", + "CONST" => "CONST", + "SINE" => "SINE", + "SQUARE" => "SQUARE", + _ => return err("RANGE", "invalid_wave"), + }); + } + "level" => match value.parse::() { + Ok(parsed) if parsed <= 4_095 => { + level = parsed; + saw_level = true; + } + _ => return err("RANGE", "invalid_level"), + }, + "min" => match value.parse::() { + Ok(parsed) if parsed <= 4_095 => min_level = parsed, + _ => return err("RANGE", "invalid_min"), + }, + "freq_mhz" => match value.parse::() { + Ok(parsed) => { + freq_mhz = parsed; + saw_freq = true; + } + _ => return err("RANGE", "invalid_freq_mhz"), + }, + _ => return err("SYNTAX", "unknown_mod_field"), + } + } + let Some(wave) = wave else { + return err("SYNTAX", "wave_required"); + }; + let periodic = wave == "SINE" || wave == "SQUARE"; + if wave != "OFF" && !saw_level { + return err("SYNTAX", "level_required"); + } + if periodic && !saw_freq { + return err("SYNTAX", "freq_mhz_required"); + } + if min_level > level { + return err("RANGE", "min_above_level"); + } + if periodic && !(MOCK_MOD_MIN_FREQ_MHZ..=MOCK_MOD_MAX_FREQ_MHZ).contains(&freq_mhz) { + return err("RANGE", "mod_rejected"); + } + if wave == "OFF" { + level = 0; + min_level = 0; + freq_mhz = 0; + } + self.mod_wave = wave; + self.mod_level = level; + self.mod_min = if wave == "CONST" { level } else { min_level }; + self.mod_freq_mhz = if periodic { freq_mhz } else { 0 }; + // Same initial output as the firmware engine: CONST/OFF hold level, + // square starts low, sine starts at the center. + self.mod_code = match wave { + "SQUARE" => self.mod_min, + "SINE" => (self.mod_min + self.mod_level) / 2, + _ => level, + }; + format!( + "+{sequence} OK mod_wave={} mod_level={} mod_min={} mod_freq_mhz={} code={}", + self.mod_wave, self.mod_level, self.mod_min, self.mod_freq_mhz, self.mod_code + ) + } + fn send_control(&mut self, payload: &str) { let frame = self.build_frame(FrameType::Control, payload.as_bytes().to_vec(), 0, 0); let bytes = frame.to_bytes(); @@ -615,20 +717,64 @@ mod tests { } #[test] - fn hello_requires_protocol_v1_and_advertises_capabilities_only_with_extension() { + fn hello_requires_protocol_v1_and_advertises_capabilities() { let link = MockLink::new(); let mut host = link.host_end(); let mut controller = MockController::new(link.device_end()); request(&mut controller, "@1 HELLO"); assert!(last_control_text(&mut host).contains("code=PROTOCOL detail=requires_v1")); request(&mut controller, "@2 HELLO protocol=1"); - assert!(!last_control_text(&mut host).contains("capabilities")); + assert!(last_control_text(&mut host).contains("capabilities=MOD,PDSTREAM")); let link = MockLink::new(); let mut host = link.host_end(); let mut controller = MockController::new(link.device_end()).with_waveform_extension(); request(&mut controller, "@1 HELLO protocol=1"); - assert!(last_control_text(&mut host).contains("capabilities=A1,A2,A3,WAVE")); + assert!(last_control_text(&mut host).contains("capabilities=MOD,PDSTREAM,WAVE")); + } + + #[test] + fn mod_command_validates_and_holds_across_stop() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + // Validation mirrors the firmware error details. + request(&mut controller, "@1 MOD level=1000"); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=wave_required")); + request(&mut controller, "@2 MOD wave=SINE level=1000"); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=freq_mhz_required")); + request( + &mut controller, + "@3 MOD wave=SQUARE level=100 min=200 freq_mhz=1000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=min_above_level")); + request( + &mut controller, + "@4 MOD wave=SINE level=1000 freq_mhz=99000000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=mod_rejected")); + + // CONST applies immediately; STATUS echoes it; STOP does not clear it. + request(&mut controller, "@5 MOD wave=CONST level=1234"); + assert!(last_control_text(&mut host) + .contains("mod_wave=CONST mod_level=1234 mod_min=1234 mod_freq_mhz=0 code=1234")); + request(&mut controller, "@6 STOP"); + request(&mut controller, "@7 STATUS"); + let status = last_control_text(&mut host); + assert!(status.contains("code=1234"), "{status}"); + assert!(status.contains("mod_wave=CONST"), "{status}"); + + // Square starts at the min threshold; OFF drops to zero. + request( + &mut controller, + "@8 MOD wave=SQUARE level=2000 min=500 freq_mhz=10000", + ); + assert!(last_control_text(&mut host) + .contains("mod_wave=SQUARE mod_level=2000 mod_min=500 mod_freq_mhz=10000 code=500")); + request(&mut controller, "@9 MOD wave=OFF"); + assert!(last_control_text(&mut host) + .contains("mod_wave=OFF mod_level=0 mod_min=0 mod_freq_mhz=0 code=0")); } #[test] From 73f4ce18e2fb167b485a6885645ede35c5e45750 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 10:31:24 +0200 Subject: [PATCH 11/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20exchange?= =?UTF-8?q?=20enum=20settings=20as=20indices=20so=20radio=20buttons=20appl?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host settings UI reads enum values with as_u64() and writes the selected variant index; the modulation/photodiode plugins returned and expected variant name strings, so port and mode radio buttons could never be set. Map indices against the schema's variant list in get_setting/set_setting (names still accepted) and pin the contract with round-trip tests. --- plugins/stage-a-modulation/src/lib.rs | 78 ++++++++++++++++++++++++--- plugins/stage-a-photodiode/src/lib.rs | 77 +++++++++++++++++++++++--- 2 files changed, 141 insertions(+), 14 deletions(-) diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index e5b163a..4c5d06f 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -420,6 +420,29 @@ fn serial_ports() -> Vec { .collect() } +/// The exact variant list the settings schema shows for the port enum — the +/// host exchanges enum settings as indices into this list. +fn port_variants() -> Vec { + let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; + variants.extend(serial_ports()); + variants +} + +/// Host enum widgets send the selected index; string names are also accepted +/// (tests, saved configs). +fn enum_choice(value: &Value, variants: &[String]) -> Result { + if let Some(index) = value.as_u64() { + return variants + .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) + .cloned() + .ok_or_else(|| format!("enum index {index} out of range")); + } + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| "expected an enum index or name".to_owned()) +} + impl Plugin for StageAModulationPlugin { fn name(&self) -> &'static str { "Stage-A Modulation" @@ -486,8 +509,7 @@ impl Plugin for StageAModulationPlugin { } fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; - port_variants.extend(serial_ports()); + let port_variants = port_variants(); let port_default = port_variants .iter() .position(|p| *p == self.port_hint) @@ -593,10 +615,24 @@ impl Plugin for StageAModulationPlugin { fn get_setting(&self, key: &str) -> Option { match key { - "port" => Some(json!(self.port_hint)), + // Enum settings are exchanged as indices into the schema's + // variant list (see the host settings UI). + "port" => { + let index = port_variants() + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + Some(json!(index)) + } "level" => Some(json!(self.level)), "max_level" => Some(json!(self.max_level)), - "mode" => Some(json!(self.mode.name())), + "mode" => { + let index = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + Some(json!(index)) + } "frequency_hz" => Some(json!(self.frequency_hz)), "min_level" => Some(json!(self.min_level)), _ => None, @@ -606,7 +642,7 @@ impl Plugin for StageAModulationPlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + self.port_hint = enum_choice(&value, &port_variants())?; Ok(()) } "level" => { @@ -636,8 +672,10 @@ impl Plugin for StageAModulationPlugin { Ok(()) } "mode" => { - let name = value.as_str().ok_or("mode must be a string")?; - self.mode = Mode::from_name(name) + let mode_names: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let name = enum_choice(&value, &mode_names)?; + self.mode = Mode::from_name(&name) .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; self.dirty = true; Ok(()) @@ -831,6 +869,32 @@ mod tests { plugin.disconnect("test done"); } + /// The host settings UI exchanges enum values as indices into the + /// schema's variant list (radio buttons send `json!(index)`). + #[test] + fn enum_settings_round_trip_as_indices() { + let mut plugin = StageAModulationPlugin::default(); + // Mode: index 2 = SQUARE in the schema's variant order. + plugin + .set_setting("mode", json!(2)) + .expect("index accepted"); + assert_eq!(plugin.mode, Mode::Square); + assert_eq!(plugin.get_setting("mode"), Some(json!(2))); + // Port: index 1 = "auto" (variants start with mock, auto). + plugin + .set_setting("port", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.port_hint, "auto"); + assert_eq!(plugin.get_setting("port"), Some(json!(1))); + // Out-of-range indices are visible errors, not silent no-ops. + assert!(plugin.set_setting("mode", json!(99)).is_err()); + // String names keep working (tests, saved configs). + plugin + .set_setting("mode", json!("SINE")) + .expect("name accepted"); + assert_eq!(plugin.mode, Mode::Sine); + } + /// min_level can never exceed the level. #[test] fn min_threshold_is_clamped_to_level() { diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 8a6f3a7..c9d1699 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -431,6 +431,29 @@ fn serial_ports() -> Vec { .unwrap_or_default() } +/// The exact variant list the settings schema shows for the port enum — the +/// host exchanges enum settings as indices into this list. +fn port_variants() -> Vec { + let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; + variants.extend(serial_ports()); + variants +} + +/// Host enum widgets send the selected index; string names are also accepted +/// (tests, saved configs). +fn enum_choice(value: &Value, variants: &[String]) -> Result { + if let Some(index) = value.as_u64() { + return variants + .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) + .cloned() + .ok_or_else(|| format!("enum index {index} out of range")); + } + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| "expected an enum index or name".to_owned()) +} + impl Plugin for StageAPhotodiodePlugin { fn name(&self) -> &'static str { "Stage-A Photodiode" @@ -490,8 +513,7 @@ impl Plugin for StageAPhotodiodePlugin { } fn settings_schema(&self) -> SettingsSchema { - let mut port_variants = vec!["mock".to_owned(), "auto".to_owned()]; - port_variants.extend(serial_ports()); + let port_variants = port_variants(); let port_default = port_variants .iter() .position(|p| *p == self.port_hint) @@ -570,8 +592,22 @@ impl Plugin for StageAPhotodiodePlugin { fn get_setting(&self, key: &str) -> Option { match key { - "port" => Some(json!(self.port_hint)), - "mode" => Some(json!(self.mode.name())), + // Enum settings are exchanged as indices into the schema's + // variant list (see the host settings UI). + "port" => { + let index = port_variants() + .iter() + .position(|p| *p == self.port_hint) + .unwrap_or(0); + Some(json!(index)) + } + "mode" => { + let index = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + Some(json!(index)) + } "reference_volts" => Some(json!(self.reference_volts)), "window_s" => Some(json!(self.window_s)), _ => None, @@ -581,12 +617,14 @@ impl Plugin for StageAPhotodiodePlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = value.as_str().ok_or("port must be a string")?.to_owned(); + self.port_hint = enum_choice(&value, &port_variants())?; Ok(()) } "mode" => { - let name = value.as_str().ok_or("mode must be a string")?; - self.mode = Mode::from_name(name) + let mode_names: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let name = enum_choice(&value, &mode_names)?; + self.mode = Mode::from_name(&name) .ok_or_else(|| format!("unknown mode: {name} (RAW/EXCITATION)"))?; Ok(()) } @@ -759,6 +797,31 @@ mod tests { plugin.disconnect(); } + /// The host settings UI exchanges enum values as indices into the + /// schema's variant list (radio buttons send `json!(index)`). + #[test] + fn enum_settings_round_trip_as_indices() { + let mut plugin = StageAPhotodiodePlugin::default(); + // Mode: index 1 = EXCITATION. + plugin + .set_setting("mode", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.mode, Mode::Excitation); + assert_eq!(plugin.get_setting("mode"), Some(json!(1))); + // Port: index 1 = "auto" (variants start with mock, auto). + plugin + .set_setting("port", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.port_hint, "auto"); + assert_eq!(plugin.get_setting("port"), Some(json!(1))); + assert!(plugin.set_setting("mode", json!(99)).is_err()); + // String names keep working (tests, saved configs). + plugin + .set_setting("mode", json!("RAW")) + .expect("name accepted"); + assert_eq!(plugin.mode, Mode::Raw); + } + #[test] fn ring_is_bounded() { let mut state = SharedState::default(); From bcb2dfcf84d7e35b747f21280f90df58595e909e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 10:43:56 +0200 Subject: [PATCH 12/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20auto-detect?= =?UTF-8?q?=20the=20correct=20Teensy=20port=20in=20both=20plugins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit auto in stage-a-modulation now probes each attached port with HELLO and picks the command port; auto in stage-a-photodiode listens for PD lines and picks the stream port. Filter macOS port lists to the cu.* callout nodes so each device appears once. Verified against the live Teensy (firmware 0.3.0). --- plugins/stage-a-modulation/README.md | 8 +-- plugins/stage-a-modulation/src/lib.rs | 52 +++++++++++++++----- plugins/stage-a-photodiode/README.md | 7 +-- plugins/stage-a-photodiode/src/lib.rs | 70 ++++++++++++++++++++++++--- 4 files changed, 110 insertions(+), 27 deletions(-) diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index 9e21a88..1b3f233 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -24,10 +24,10 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** ## Ports -Select the Teensy *command* port (binary protocol), not the photodiode stream port. `mock` runs an -in-process simulated controller for hardware-free testing; `auto` picks the first -usbmodem/ttyACM device. If you picked the wrong physical port, HELLO simply times out — pick the -other one. +**Use `auto` (default recommendation):** it probes every attached usbmodem/ttyACM device and +connects to the one that answers `HELLO` — that is always the Teensy command port, never the +photodiode stream port. Explicit ports remain selectable; `mock` runs an in-process simulated +controller for hardware-free testing. Hardware commands only flow while the host execution context allows effects (live capture); otherwise the connection is torn down and the panel shows the lock reason. diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 4c5d06f..a31f72a 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -399,24 +399,51 @@ impl StageAModulationPlugin { } fn open_serial(port_hint: &str) -> Result, String> { - let path = if port_hint == "auto" { - serial_ports() - .into_iter() - .next() - .ok_or_else(|| "no USB serial device found (looked for usbmodem/ttyACM)".to_owned())? - } else { - port_hint.to_owned() - }; + if port_hint == "auto" { + // The dual-serial Teensy enumerates two ports and only the command + // port answers HELLO — probe until one does. + let candidates = serial_ports(); + if candidates.is_empty() { + return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + } + let mut failures = Vec::new(); + for path in &candidates { + match probe_command_port(path) { + // Restore the client's default reply timeout after probing. + Ok(client) => return Ok(client.with_reply_timeout(Duration::from_millis(500))), + Err(err) => failures.push(format!("{path}: {err}")), + } + } + return Err(format!( + "no Teensy command port answered HELLO ({})", + failures.join("; ") + )); + } + open_path(port_hint) +} + +fn open_path(path: &str) -> Result, String> { let transport = - stage_a_io::SerialTransport::open(&path, 115_200, std::time::Duration::from_millis(20)) + stage_a_io::SerialTransport::open(path, 115_200, std::time::Duration::from_millis(20)) .map_err(|err| err.to_string())?; Ok(StageAClient::new(transport)) } +/// Opens `path` and sends HELLO with a short timeout: only the Teensy +/// command port replies (the photodiode stream port never answers). +fn probe_command_port(path: &str) -> Result, String> { + let mut client = open_path(path)?.with_reply_timeout(Duration::from_millis(300)); + client + .request(&Command::new("HELLO").field("protocol", 1)) + .map_err(|err| err.to_string())?; + Ok(client) +} + fn serial_ports() -> Vec { stage_a_io::transport::available_port_names() .into_iter() - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + // macOS lists each device twice; use the callout (cu.*) node only. + .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) .collect() } @@ -535,8 +562,9 @@ impl Plugin for StageAModulationPlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "Teensy command port (the FIRST of the two usbmodem ports); \ - mock = in-process simulated controller, auto = first device" + "auto (recommended) probes the attached usbmodem ports and picks \ + the one that answers HELLO — the Teensy command port; \ + mock = in-process simulated controller" .into(), ), kind: SettingKind::Enum { diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index 5f59e82..af2e277 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -20,6 +20,7 @@ read-only by construction; the command port belongs to `stage-a-modulation`. ## Ports -Select the Teensy *stream* port (the second `usbmodem` port). Picking the command port by mistake -is harmless: its binary frames simply parse to nothing (no values appear) — switch to the other -port. `mock` generates a synthetic slow sine for hardware-free testing. +**Use `auto` (default recommendation):** it listens briefly on every attached usbmodem/ttyACM +device and connects to the one actually streaming `PD` lines — that is always the Teensy stream +port. Picking the command port manually by mistake is harmless: its binary frames parse to +nothing (no values appear). `mock` generates a synthetic slow sine for hardware-free testing. diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index c9d1699..49fb4c9 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -275,11 +275,10 @@ impl StageAPhotodiodePlugin { return; } let path = if self.port_hint == "auto" { - match serial_ports().into_iter().next() { - Some(path) => path, - None => { - self.last_error = - Some("no USB serial device found (looked for usbmodem/ttyACM)".into()); + match resolve_auto_port() { + Ok(path) => path, + Err(err) => { + self.last_error = Some(err); return; } } @@ -425,12 +424,66 @@ fn serial_ports() -> Vec { ports .into_iter() .map(|p| p.port_name) - .filter(|name| name.contains("usbmodem") || name.contains("ttyACM")) + // macOS lists each device twice; use the callout (cu.*) node only. + .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) .collect() }) .unwrap_or_default() } +/// Finds the Teensy stream port: the dual-serial firmware free-runs `PD` +/// lines on exactly one of the enumerated ports, so listen briefly on each. +fn resolve_auto_port() -> Result { + let candidates = serial_ports(); + if candidates.is_empty() { + return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + } + for path in &candidates { + if probe_pd_stream(path) { + return Ok(path.clone()); + } + } + Err(format!( + "no port streamed PD lines within 500 ms (tried {})", + candidates.join(", ") + )) +} + +/// True when `path` produces a parsable `PD …` line within the probe window. +fn probe_pd_stream(path: &str) -> bool { + let Ok(mut port) = serialport::new(path, 115_200) + .timeout(Duration::from_millis(100)) + .open() + else { + return false; + }; + let deadline = Instant::now() + Duration::from_millis(500); + let mut collected: Vec = Vec::new(); + let mut buf = [0_u8; 512]; + while Instant::now() < deadline { + match port.read(&mut buf) { + Ok(read) if read > 0 => { + collected.extend_from_slice(&buf[..read]); + if String::from_utf8_lossy(&collected) + .lines() + .any(|line| parse_pd_line(line).is_some()) + { + return true; + } + if collected.len() > 8_192 { + collected.drain(..4_096); + } + } + Ok(_) => {} + Err(err) + if err.kind() == std::io::ErrorKind::TimedOut + || err.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return false, + } + } + false +} + /// The exact variant list the settings schema shows for the port enum — the /// host exchanges enum settings as indices into this list. fn port_variants() -> Vec { @@ -539,8 +592,9 @@ impl Plugin for StageAPhotodiodePlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "Teensy stream port (the SECOND usbmodem port); mock = synthetic \ - data, auto = first device" + "auto (recommended) listens on the attached usbmodem ports and \ + picks the one streaming PD lines — the Teensy stream port; \ + mock = synthetic data" .into(), ), kind: SettingKind::Enum { From cc4b435d24f98f387e32c9341952601444098b0a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 10:55:28 +0200 Subject: [PATCH 13/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20label=20port?= =?UTF-8?q?=20choices=20with=20their=20USB=20product=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port entries now read '/dev/cu.usbmodem… (Teensyduino Dual Serial)' so the Teensy is recognisable among the enumerated devices; the parenthesised label is display-only and stripped when the setting is applied. stage-a-io gains available_ports_with_labels() for this. --- plugins/stage-a-modulation/src/lib.rs | 25 ++++++++++++++---- plugins/stage-a-photodiode/src/lib.rs | 32 ++++++++++++++++++++--- stage-a-io/src/transport.rs | 37 +++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 9 deletions(-) diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index a31f72a..b941a9d 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -448,13 +448,28 @@ fn serial_ports() -> Vec { } /// The exact variant list the settings schema shows for the port enum — the -/// host exchanges enum settings as indices into this list. +/// host exchanges enum settings as indices into this list. Real ports carry +/// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; +/// only the leading path is the value. fn port_variants() -> Vec { let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; - variants.extend(serial_ports()); + for (name, label) in stage_a_io::transport::available_ports_with_labels() { + if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { + continue; + } + variants.push(match label { + Some(label) => format!("{name} ({label})"), + None => name, + }); + } variants } +/// The path part of a port variant; the parenthesised USB label is display-only. +fn variant_path(variant: &str) -> &str { + variant.split_whitespace().next().unwrap_or(variant) +} + /// Host enum widgets send the selected index; string names are also accepted /// (tests, saved configs). fn enum_choice(value: &Value, variants: &[String]) -> Result { @@ -539,7 +554,7 @@ impl Plugin for StageAModulationPlugin { let port_variants = port_variants(); let port_default = port_variants .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); let mode_variants: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); @@ -648,7 +663,7 @@ impl Plugin for StageAModulationPlugin { "port" => { let index = port_variants() .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); Some(json!(index)) } @@ -670,7 +685,7 @@ impl Plugin for StageAModulationPlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = enum_choice(&value, &port_variants())?; + self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } "level" => { diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 49fb4c9..d6f2928 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -488,10 +488,34 @@ fn probe_pd_stream(path: &str) -> bool { /// host exchanges enum settings as indices into this list. fn port_variants() -> Vec { let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; - variants.extend(serial_ports()); + for port in serialport::available_ports().unwrap_or_default() { + if !(port.port_name.contains("cu.usbmodem") || port.port_name.contains("ttyACM")) { + continue; + } + let label = match port.port_type { + serialport::SerialPortType::UsbPort(info) => match (info.manufacturer, info.product) { + (Some(manufacturer), Some(product)) if !product.starts_with(&manufacturer) => { + Some(format!("{manufacturer} {product}")) + } + (_, Some(product)) => Some(product), + (Some(manufacturer), None) => Some(manufacturer), + (None, None) => None, + }, + _ => None, + }; + variants.push(match label { + Some(label) => format!("{} ({label})", port.port_name), + None => port.port_name, + }); + } variants } +/// The path part of a port variant; the parenthesised USB label is display-only. +fn variant_path(variant: &str) -> &str { + variant.split_whitespace().next().unwrap_or(variant) +} + /// Host enum widgets send the selected index; string names are also accepted /// (tests, saved configs). fn enum_choice(value: &Value, variants: &[String]) -> Result { @@ -569,7 +593,7 @@ impl Plugin for StageAPhotodiodePlugin { let port_variants = port_variants(); let port_default = port_variants .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); let mode_variants: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); @@ -651,7 +675,7 @@ impl Plugin for StageAPhotodiodePlugin { "port" => { let index = port_variants() .iter() - .position(|p| *p == self.port_hint) + .position(|p| variant_path(p) == self.port_hint) .unwrap_or(0); Some(json!(index)) } @@ -671,7 +695,7 @@ impl Plugin for StageAPhotodiodePlugin { fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { match key { "port" => { - self.port_hint = enum_choice(&value, &port_variants())?; + self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } "mode" => { diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index fc8482a..f7e677a 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -125,3 +125,40 @@ pub fn available_port_names() -> Vec { pub fn available_port_names() -> Vec { Vec::new() } + +/// Port names plus a human-readable USB label (manufacturer/product) where +/// the OS provides one — e.g. `("/dev/cu.usbmodem…", Some("Teensyduino Dual +/// Serial"))`. Lets port pickers show which entry is the Teensy. +#[cfg(feature = "hardware")] +pub fn available_ports_with_labels() -> Vec<(String, Option)> { + serialport::available_ports() + .map(|ports| { + ports + .into_iter() + .map(|p| { + let label = match p.port_type { + serialport::SerialPortType::UsbPort(info) => { + match (info.manufacturer, info.product) { + (Some(manufacturer), Some(product)) + if !product.starts_with(&manufacturer) => + { + Some(format!("{manufacturer} {product}")) + } + (_, Some(product)) => Some(product), + (Some(manufacturer), None) => Some(manufacturer), + (None, None) => None, + } + } + _ => None, + }; + (p.port_name, label) + }) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(not(feature = "hardware"))] +pub fn available_ports_with_labels() -> Vec<(String, Option)> { + Vec::new() +} From 449f758b4d734f0403fe8c0bc2bb119bcc8777b4 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 11:21:25 +0200 Subject: [PATCH 14/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20make=20dev?= =?UTF-8?q?ice=20control=20settings-driven=20so=20it=20works=20without=20c?= =?UTF-8?q?amera=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host only calls process_frame() while camera frames flow, so the action-button + effects-gate design never connected on a camera-less bench (verified: the serial port stayed free while the GUI ran). Connect is now a checkbox setting handled in set_setting, all serial I/O lives in a plugin-owned device thread (slider drags coalesce into one pending MOD), status comes from shared state, and replay mode still disconnects defensively. Verified end-to-end against the live Teensy: auto-probe, level 800 -> board code 800, level 0 -> 0. --- docs/adr/006-stage-a-two-plugin-split.md | 10 +- docs/features/stage-a-modulation.md | 12 +- docs/features/stage-a-photodiode.md | 3 +- plugins/stage-a-modulation/README.md | 17 +- plugins/stage-a-modulation/src/lib.rs | 646 ++++++++++++----------- plugins/stage-a-photodiode/src/lib.rs | 123 ++--- 6 files changed, 406 insertions(+), 405 deletions(-) diff --git a/docs/adr/006-stage-a-two-plugin-split.md b/docs/adr/006-stage-a-two-plugin-split.md index 99971e3..618e348 100644 --- a/docs/adr/006-stage-a-two-plugin-split.md +++ b/docs/adr/006-stage-a-two-plugin-split.md @@ -31,10 +31,12 @@ control plugin's connection. mock — the mock now models firmware 0.3.0's `MOD` verb). The A1/A2/A3 experiment plugins will build on it again when the bench reaches that stage; the estimator/pdq/sidecar modules remain for that purpose even though no current plugin uses them. -4. **Immediate transfer replaces the Apply-action pattern** in `stage-a-modulation`: setting - changes are sent to the device as they happen (the operator's explicit request), still behind - the fail-closed execution-context gate. The firmware output is set-and-hold; the explicit - "Output OFF" action is the only stop. +4. **Immediate transfer replaces the Apply-action pattern**, and **all device control is + settings-driven** (connect checkbox, slider changes sent as they happen). Host actions and + the per-frame effects gate are unsuitable here: the host only runs `process_frame()` while + camera frames flow, but the bench must work with no camera attached (amended 2026-07-16). + Replay mode still disconnects the modulation plugin defensively. The firmware output is + set-and-hold; the power slider at 0 is the off switch. ## Consequences diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index c495d9a..49279fa 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -20,12 +20,14 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val - Owns the Teensy **command port** exclusively (one owner per port, ADR 006). The photodiode stream port belongs to `stage-a-photodiode`. -- Uses `stage-a-io` (`StageAClient`, `IoWorker`, `Command`) for framing, idempotent retries, and - the bounded background I/O thread; `process_frame()` never blocks on serial. -- Fail-closed effects gate: the connection only exists while the host execution context allows - hardware effects. +- Uses `stage-a-io` (`StageAClient`, `Command`) for framing and idempotent retries; slider drags + coalesce into a single pending command the device thread drains. +- **Frame-independent**: connecting is a checkbox setting and all serial I/O lives in a + plugin-owned device thread, because the host only calls `process_frame()` while camera frames + flow — bench control must work with no camera attached. `process_frame()` only disconnects + defensively in replay mode. - Firmware output is **set-and-hold** (`stage-a-controller` ADR 002): disconnecting does not stop - the modulation. The explicit **Output OFF** action sends `MOD wave=OFF`. + the modulation. The power slider at 0 is the off switch. - Safety invariants enforced plugin-side: `level ≤ max_level`, `min_level ≤ level`; the firmware waveform peaks at `level` by construction. - `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 976b9a8..916ce8c 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -23,7 +23,8 @@ Two modes: - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the plugin is read-only by construction and needs no protocol library — it depends only on `serialport` and parses one line format. -- Same fail-closed effects gate as the other stage-a plugins for consistent device handling. +- **Frame-independent**: connecting is a checkbox setting; the reader thread and all views + work with no camera attached (the host only calls `process_frame()` while frames flow). - Garbage on the port (e.g. the binary command port picked by mistake) parses to nothing and is bounded — it can neither grow memory nor produce fake values. - `mock` port synthesizes a slow sine for hardware-free testing. diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index 1b3f233..b349ba6 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -14,13 +14,14 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** - The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and a 2 Hz `STATUS` poll), not just what was commanded. -## Actions +## Connecting -- **Connect / Disconnect** — open/close the command port. Connecting never changes the output; - only changes made while connected are transferred. -- **Output OFF** — sends `MOD wave=OFF` (DAC code 0). Needed because the firmware output is - **set-and-hold**: disconnecting, closing the GUI, or a crash leaves the last modulation running - (`stage-a-controller` ADR 002). +- **Connect** is a checkbox in the plugin settings — it opens/closes the command port and works + **without a running camera** (device I/O lives in a plugin-owned thread, independent of the + host's frame-driven plugin passes). Connecting never changes the output; only changes made + while connected are transferred. +- **Output off = power slider at 0.** The firmware output is **set-and-hold**: disconnecting, + closing the GUI, or a crash leaves the last modulation running (`stage-a-controller` ADR 002). ## Ports @@ -29,5 +30,5 @@ connects to the one that answers `HELLO` — that is always the Teensy command p photodiode stream port. Explicit ports remain selectable; `mock` runs an in-process simulated controller for hardware-free testing. -Hardware commands only flow while the host execution context allows effects (live capture); -otherwise the connection is torn down and the panel shows the lock reason. +Replaying a recording disconnects the plugin defensively; live control itself needs no +capture session. diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index b941a9d..87fc75f 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -6,45 +6,41 @@ //! with frequency and a lower threshold for the periodic modes — and every //! accepted change is transferred to the Teensy immediately, no Apply button. //! -//! The plugin owns the Teensy **command port** (the first of the two CDC -//! ports the dual-serial firmware enumerates; the photodiode stream port is -//! owned by `stage-a-photodiode`). The firmware output is set-and-hold: -//! disconnecting does NOT switch the modulation off — use the "Output OFF" -//! action (ADR 002 in `stage-a-controller`). +//! **Frame-independent by design.** The host only calls `process_frame()` +//! while camera frames flow, so nothing here depends on it: connecting is a +//! checkbox *setting* (settings arrive from the UI thread at any time), a +//! dedicated device thread owns the serial client, and slider changes are +//! coalesced into a pending-command slot that thread drains. The bench works +//! with no camera attached. `process_frame()` only tears the connection down +//! defensively in replay mode. //! -//! Safety contract: -//! - devices open only when the execution context allows hardware effects; -//! anything else tears the connection down (fail closed); -//! - the level slider cannot exceed the max-level cap, and the firmware -//! output can never exceed the slider (square/sine peak at `level`); -//! - `process_frame()` only drains the bounded I/O worker queues. +//! The plugin owns the Teensy **command port**; the photodiode stream port is +//! owned by `stage-a-photodiode`. The firmware output is set-and-hold +//! (`stage-a-controller` ADR 002): disconnecting does NOT switch the +//! modulation off — drag the power slider to 0 to drive 0 V. use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, SettingItem, - SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, - TableColumnValues, TableDatasetV1, TableSchema, TableValueType, - CTX_INVESTIGATION_ACTION_REQUESTS, + export_plugin, EventStoreHandle, ExecutionMode, HostContext, HostDatasetDescriptor, + HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, + HostViewRegistry, Plugin, PluginFrame, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{Command, IoWorker, MockController, StageAClient, WorkerOutput, WorkerRequest}; +use stage_a_io::{Command, MockController, StageAClient, Transport}; const STATUS_DATASET_ID: &str = "stage-a-modulation.status"; const STATUS_VIEW_ID: &str = "stage-a-modulation.status.view"; -const ACTION_CONNECT: &str = "stage-a-modulation.connect"; -const ACTION_DISCONNECT: &str = "stage-a-modulation.disconnect"; -const ACTION_OUTPUT_OFF: &str = "stage-a-modulation.output-off"; - const MAX_DAC_CODE: i64 = 4_095; const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); +const DEVICE_LOOP_TICK: Duration = Duration::from_millis(10); #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Mode { @@ -73,6 +69,41 @@ impl Mode { } } +/// State the device thread reports back for the UI (status entries, table). +#[derive(Default)] +struct DeviceState { + connected: bool, + firmware: String, + board_code: Option, + board_mod: String, + last_error: Option, +} + +/// Everything shared between the plugin (UI thread) and the device thread. +struct SharedLink { + state: Mutex, + /// Latest not-yet-sent command; newer settings overwrite older ones so + /// slider drags coalesce instead of queueing. + pending: Mutex>, + stop: AtomicBool, + generation: AtomicU64, +} + +impl SharedLink { + fn new() -> Self { + Self { + state: Mutex::new(DeviceState::default()), + pending: Mutex::new(None), + stop: AtomicBool::new(false), + generation: AtomicU64::new(1), + } + } + + fn bump(&self) { + self.generation.fetch_add(1, Ordering::Relaxed); + } +} + /// In-process mock controller thread behind the `mock` port. struct MockService { stop: Arc, @@ -113,125 +144,200 @@ impl Drop for MockService { } } +/// Handle to the running device thread; dropping it stops the thread. +struct DeviceLink { + shared: Arc, + join: Option>, + _mock: Option, +} + +impl Drop for DeviceLink { + fn drop(&mut self) { + self.shared.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +/// Device thread: HELLO once, then drain the pending command slot and poll +/// STATUS. All serial I/O lives here — the UI thread never blocks. +fn run_device(mut client: StageAClient, shared: Arc) { + match client.request(&Command::new("HELLO").field("protocol", 1)) { + Ok(fields) => { + let mut state = shared.state.lock().expect("device state lock"); + state.connected = true; + state.firmware = fields + .get("firmware") + .cloned() + .unwrap_or_else(|| "unknown".into()); + let has_mod = fields + .get("capabilities") + .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); + state.last_error = (!has_mod).then(|| { + "firmware has no MOD capability — flash stage-a-controller 0.3.0+".to_owned() + }); + } + Err(err) => { + let mut state = shared.state.lock().expect("device state lock"); + state.connected = false; + state.last_error = Some(format!("HELLO failed: {err}")); + shared.bump(); + return; + } + } + shared.bump(); + + let mut last_status = Instant::now() - STATUS_POLL_INTERVAL; + while !shared.stop.load(Ordering::Relaxed) { + let pending = shared.pending.lock().expect("pending lock").take(); + if let Some(command) = pending { + let result = client.request(&command); + apply_reply(&shared, "MOD", result); + } else if last_status.elapsed() >= STATUS_POLL_INTERVAL { + last_status = Instant::now(); + let result = client.request(&Command::new("STATUS")); + apply_reply(&shared, "STATUS", result); + } else { + std::thread::sleep(DEVICE_LOOP_TICK); + } + } + + let mut state = shared.state.lock().expect("device state lock"); + state.connected = false; + shared.bump(); +} + +fn apply_reply( + shared: &SharedLink, + purpose: &str, + result: Result, stage_a_io::ClientError>, +) { + let mut state = shared.state.lock().expect("device state lock"); + match result { + Ok(fields) => { + if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { + state.board_code = Some(code); + } + if let Some(wave) = fields.get("mod_wave") { + let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); + let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); + let freq_mhz = fields + .get("mod_freq_mhz") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + state.board_mod = if wave == "SINE" || wave == "SQUARE" { + format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) + } else { + format!("{wave} level={level}") + }; + } + if purpose == "MOD" { + state.last_error = None; + } + } + Err(err) => state.last_error = Some(format!("{purpose}: {err}")), + } + drop(state); + shared.bump(); +} + pub struct StageAModulationPlugin { enabled: bool, - // -- device -- - worker: Option, - mock_service: Option, - connected: bool, - firmware: String, - next_tag: u64, - in_flight: BTreeMap, - last_error: Option, - effects_blocked_reason: Option, - last_status_poll: Instant, + link: Option, + shared: Arc, // -- settings (every accepted change is sent immediately) -- + connect_requested: bool, port_hint: String, max_level: i64, level: i64, min_level: i64, mode: Mode, frequency_hz: f64, - dirty: bool, - // -- board-reported state (from MOD replies and STATUS polls) -- - board_code: Option, - board_mod: String, - dataset_generation: u64, - consumed_action_ids: Vec, + last_error: Option, } impl Default for StageAModulationPlugin { fn default() -> Self { Self { enabled: false, - worker: None, - mock_service: None, - connected: false, - firmware: String::new(), - next_tag: 1, - in_flight: BTreeMap::new(), - last_error: None, - effects_blocked_reason: None, - last_status_poll: Instant::now(), - port_hint: "mock".into(), + link: None, + shared: Arc::new(SharedLink::new()), + connect_requested: false, + port_hint: "auto".into(), max_level: MAX_DAC_CODE, level: 0, min_level: 0, mode: Mode::Const, frequency_hz: 10.0, - dirty: false, - board_code: None, - board_mod: "—".into(), - dataset_generation: 0, - consumed_action_ids: Vec::new(), + last_error: None, } } } impl StageAModulationPlugin { - fn bump_generation(&mut self) { - self.dataset_generation = self.dataset_generation.wrapping_add(1); - } - - fn queue_command(&mut self, purpose: &str, command: Command) { - let Some(worker) = &self.worker else { - self.last_error = Some(format!("{purpose}: no device connection")); - return; - }; - let tag = self.next_tag; - self.next_tag += 1; - match worker.try_send(WorkerRequest::Send { tag, command }) { - Ok(()) => { - self.in_flight.insert(tag, purpose.to_owned()); - } - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - fn connect(&mut self) { - if self.worker.is_some() { + if self.link.is_some() { return; } + self.last_error = None; + *self.shared.state.lock().expect("device state lock") = DeviceState::default(); + *self.shared.pending.lock().expect("pending lock") = None; + self.shared.stop.store(false, Ordering::Relaxed); + self.shared.bump(); + + let shared = Arc::clone(&self.shared); + let spawn = |name: &str, f: Box| { + std::thread::Builder::new() + .name(name.to_owned()) + .spawn(f) + .expect("spawning the device thread must succeed") + }; if self.port_hint == "mock" { - let (service, client) = MockService::spawn(); - self.mock_service = Some(service); - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; + let (mock, client) = MockService::spawn(); + let join = spawn( + "stage-a-modulation-device", + Box::new(move || run_device(client, shared)), + ); + self.link = Some(DeviceLink { + shared: Arc::clone(&self.shared), + join: Some(join), + _mock: Some(mock), + }); } else { match open_serial(&self.port_hint) { Ok(client) => { - self.worker = Some(IoWorker::spawn(client)); - self.last_error = None; + let join = spawn( + "stage-a-modulation-device", + Box::new(move || run_device(client, shared)), + ); + self.link = Some(DeviceLink { + shared: Arc::clone(&self.shared), + join: Some(join), + _mock: None, + }); } Err(err) => { self.last_error = Some(err); - return; + self.connect_requested = false; } } } - // Connecting never drives the output: only changes made while - // connected are transferred. - self.dirty = false; - self.queue_command("hello", Command::new("HELLO").field("protocol", 1)); - self.bump_generation(); + // Connecting never drives the output (set-and-hold firmware); only + // changes made while connected are transferred. } - fn disconnect(&mut self, reason: &str) { - if let Some(worker) = self.worker.take() { - worker.shutdown(reason); - } - self.mock_service = None; - self.connected = false; - self.firmware.clear(); - self.in_flight.clear(); - self.board_code = None; - self.board_mod = "—".into(); - self.bump_generation(); + fn disconnect(&mut self) { + self.link = None; // Drop stops and joins the device thread. + self.shared.bump(); } - /// One MOD command carrying the complete current drive settings. + /// Queues one MOD command carrying the complete current drive settings; + /// newer changes overwrite queued ones (drag coalescing). fn send_modulation(&mut self) { - self.dirty = false; + if self.link.is_none() { + return; + } let level = self.level.clamp(0, self.max_level); let mut command = Command::new("MOD") .field("wave", self.mode.name()) @@ -242,103 +348,16 @@ impl StageAModulationPlugin { .field("min", self.min_level.clamp(0, level)) .field("freq_mhz", freq_mhz); } - self.queue_command("mod", command); - } - - fn output_off(&mut self) { - self.dirty = false; - self.queue_command("mod", Command::new("MOD").field("wave", "OFF")); - } - - fn drain_worker(&mut self) { - let Some(worker) = &self.worker else { - return; - }; - let outputs = worker.drain_outputs(); - if outputs.is_empty() { - return; - } - let mut stopped: Option = None; - for output in outputs { - match output { - WorkerOutput::Reply { tag, result } => { - let purpose = self.in_flight.remove(&tag).unwrap_or_default(); - match result { - Ok(fields) => self.handle_reply(&purpose, &fields), - Err(err) => self.last_error = Some(format!("{purpose}: {err}")), - } - } - WorkerOutput::Event(_) | WorkerOutput::Integrity(_) => {} - WorkerOutput::Stopped { reason } => stopped = Some(reason), - } - } - if let Some(reason) = stopped { - self.worker = None; - self.mock_service = None; - self.connected = false; - self.last_error = Some(format!("device connection ended: {reason}")); - } - self.bump_generation(); - } - - fn handle_reply(&mut self, purpose: &str, fields: &BTreeMap) { - if purpose == "hello" { - self.firmware = fields - .get("firmware") - .cloned() - .unwrap_or_else(|| "unknown".into()); - self.connected = true; - let has_mod = fields - .get("capabilities") - .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); - if !has_mod { - self.last_error = - Some("firmware has no MOD capability — flash stage-a-controller 0.3.0+".into()); - } - } - // MOD replies and STATUS polls both carry code= and mod_* fields. - if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { - self.board_code = Some(code); - } - if let Some(wave) = fields.get("mod_wave") { - let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); - let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); - let freq_mhz = fields - .get("mod_freq_mhz") - .and_then(|v| v.parse::().ok()) - .unwrap_or(0.0); - self.board_mod = if wave == "SINE" || wave == "SQUARE" { - format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) - } else { - format!("{wave} level={level}") - }; - } - if purpose == "mod" { - self.last_error = None; - } + *self.shared.pending.lock().expect("pending lock") = Some(command); } - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-modulation.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed + #[cfg(test)] + fn device_connected(&self) -> bool { + self.shared + .state + .lock() + .map(|state| state.connected) + .unwrap_or(false) } fn commanded_summary(&self) -> String { @@ -356,25 +375,39 @@ impl StageAModulationPlugin { } fn status_dataset(&self) -> TableDatasetV1 { - let state = match (&self.effects_blocked_reason, self.connected) { - (Some(reason), _) => format!("locked ({reason})"), - (None, false) => "disconnected".into(), - (None, true) => format!("connected ({})", self.firmware), + let state = self.shared.state.lock().expect("device state lock"); + let connection = if state.connected { + format!("connected ({})", state.firmware) + } else if self.connect_requested { + "connecting…".into() + } else { + "disconnected".into() }; - let board_code = self + let board_code = state .board_code .map_or_else(|| "—".into(), |code| code.to_string()); + let error = state + .last_error + .clone() + .or_else(|| self.last_error.clone()) + .unwrap_or_default(); + let board_mod = if state.board_mod.is_empty() { + "—".to_owned() + } else { + state.board_mod.clone() + }; + drop(state); let text_column = |id: &str, value: String| TableColumnData { column_id: id.to_owned(), values: TableColumnValues::String(vec![value]), }; TableDatasetV1 { columns: vec![ - text_column("state", state), + text_column("state", connection), text_column("commanded", self.commanded_summary()), - text_column("board_mod", self.board_mod.clone()), + text_column("board_mod", board_mod), text_column("board_code", board_code), - text_column("error", self.last_error.clone().unwrap_or_default()), + text_column("error", error), ], } } @@ -452,7 +485,7 @@ fn serial_ports() -> Vec { /// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; /// only the leading path is the value. fn port_variants() -> Vec { - let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; + let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; for (name, label) in stage_a_io::transport::available_ports_with_labels() { if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { continue; @@ -501,13 +534,12 @@ impl Plugin for StageAModulationPlugin { fn set_enabled(&mut self, enabled: bool) { self.enabled = enabled; if !enabled { - self.disconnect("plugin disabled"); + self.connect_requested = false; + self.disconnect(); } } - fn reset(&mut self) { - self.bump_generation(); - } + fn reset(&mut self) {} fn process_frame( &mut self, @@ -516,38 +548,14 @@ impl Plugin for StageAModulationPlugin { context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - // Fail closed: without live-capture effects the connection is torn - // down and no command leaves the plugin. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.worker.is_some() { - self.disconnect("execution context revoked effects"); - } - return; - } - self.effects_blocked_reason = None; - - for action_id in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect("operator"), - ACTION_OUTPUT_OFF => self.output_off(), - _ => {} - } + // Control is settings-driven and works without camera frames. The + // only frame-pass policy: replaying a recording must never keep a + // hardware connection alive. + if context.execution().mode == ExecutionMode::Replay && self.link.is_some() { + self.connect_requested = false; + self.disconnect(); + self.last_error = Some("disconnected: replay mode".into()); } - - if self.dirty && self.connected { - self.send_modulation(); - } - if self.connected && self.last_status_poll.elapsed() >= STATUS_POLL_INTERVAL { - self.last_status_poll = Instant::now(); - self.queue_command("status", Command::new("STATUS")); - } - self.drain_worker(); } fn settings_schema(&self) -> SettingsSchema { @@ -566,9 +574,10 @@ impl Plugin for StageAModulationPlugin { sections: vec![SettingsSection { label: "Laser modulation".into(), description: Some( - "Every change is sent to the Teensy immediately. The output never exceeds \ - the power slider, and the slider never exceeds the max limit. The firmware \ - holds the output when the plugin disconnects — use Output OFF to drive 0." + "Tick Connect, then every change is sent to the Teensy immediately — no \ + camera required. The output never exceeds the power slider, the slider \ + never exceeds the max limit. The firmware holds the output when \ + disconnected; drag the slider to 0 to drive 0 V." .into(), ), default_open: true, @@ -587,12 +596,24 @@ impl Plugin for StageAModulationPlugin { default: port_default, }, }, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the command port. Connecting never changes the \ + output; disconnecting leaves it held (set-and-hold firmware)." + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, + }, SettingItem { key: "level".into(), label: "Power (DAC code)".into(), tooltip: Some( "Output level in DAC codes; peak value for sine/square. \ - Capped by the max limit below." + Capped by the max limit below. 0 = output off." .into(), ), kind: SettingKind::I64Slider { @@ -667,6 +688,7 @@ impl Plugin for StageAModulationPlugin { .unwrap_or(0); Some(json!(index)) } + "connect" => Some(json!(self.connect_requested)), "level" => Some(json!(self.level)), "max_level" => Some(json!(self.max_level)), "mode" => { @@ -688,6 +710,16 @@ impl Plugin for StageAModulationPlugin { self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } + "connect" => { + let requested = value.as_bool().ok_or("connect must be a boolean")?; + self.connect_requested = requested; + if requested { + self.connect(); + } else { + self.disconnect(); + } + Ok(()) + } "level" => { self.level = value .as_i64() @@ -696,7 +728,7 @@ impl Plugin for StageAModulationPlugin { if self.min_level > self.level { self.min_level = self.level; } - self.dirty = true; + self.send_modulation(); Ok(()) } "max_level" => { @@ -707,7 +739,7 @@ impl Plugin for StageAModulationPlugin { // Lowering the cap below the current level lowers the output. if self.level > self.max_level { self.level = self.max_level; - self.dirty = true; + self.send_modulation(); } if self.min_level > self.max_level { self.min_level = self.max_level; @@ -720,14 +752,14 @@ impl Plugin for StageAModulationPlugin { let name = enum_choice(&value, &mode_names)?; self.mode = Mode::from_name(&name) .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; - self.dirty = true; + self.send_modulation(); Ok(()) } "frequency_hz" => { let hz = value.as_f64().ok_or("frequency_hz must be a number")?; self.frequency_hz = hz.clamp(0.01, 2_000.0); if self.mode.is_periodic() { - self.dirty = true; + self.send_modulation(); } Ok(()) } @@ -737,7 +769,7 @@ impl Plugin for StageAModulationPlugin { .ok_or("min_level must be an integer")? .clamp(0, self.level); if self.mode.is_periodic() { - self.dirty = true; + self.send_modulation(); } Ok(()) } @@ -747,35 +779,27 @@ impl Plugin for StageAModulationPlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } - entries.push(StatusEntry::Text(if self.connected { - format!("Modulation: connected ({})", self.firmware) + let state = self.shared.state.lock().expect("device state lock"); + entries.push(StatusEntry::Text(if state.connected { + format!("Modulation: connected ({})", state.firmware) + } else if self.connect_requested { + "Modulation: connecting…".into() } else { "Modulation: disconnected".into() })); - if let Some(code) = self.board_code { + if let Some(code) = state.board_code { entries.push(StatusEntry::Text(format!( "Board: code={code} ({})", - self.board_mod + state.board_mod ))); } - if let Some(error) = &self.last_error { + if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } entries } fn host_views(&self) -> HostViewRegistry { - let action = |id: &str, title: &str| HostActionDescriptor { - id: id.into(), - title: title.into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }; HostViewRegistry { datasets: vec![HostDatasetDescriptor { id: STATUS_DATASET_ID.into(), @@ -792,11 +816,7 @@ impl Plugin for StageAModulationPlugin { placement: HostViewPlacement::AnalysisPanel, kind: HostViewKind::CompactTable, }], - actions: vec![ - action(ACTION_CONNECT, "Connect"), - action(ACTION_DISCONNECT, "Disconnect"), - action(ACTION_OUTPUT_OFF, "Output OFF"), - ], + actions: Vec::new(), } } @@ -809,7 +829,7 @@ impl Plugin for StageAModulationPlugin { fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { match dataset_id { - STATUS_DATASET_ID => self.dataset_generation.max(1), + STATUS_DATASET_ID => self.shared.generation.load(Ordering::Relaxed).max(1), _ => 0, } } @@ -817,7 +837,7 @@ impl Plugin for StageAModulationPlugin { impl Drop for StageAModulationPlugin { fn drop(&mut self) { - self.disconnect("plugin destroyed"); + self.disconnect(); } } @@ -827,14 +847,13 @@ export_plugin!(StageAModulationPlugin); mod tests { use super::*; - fn drain_until bool>( - plugin: &mut StageAModulationPlugin, + fn wait_until bool>( + plugin: &StageAModulationPlugin, timeout: Duration, - mut done: F, + done: F, ) { let deadline = Instant::now() + timeout; while Instant::now() < deadline { - plugin.drain_worker(); if done(plugin) { return; } @@ -843,25 +862,31 @@ mod tests { panic!("condition not reached within {timeout:?}"); } - /// Slider change → MOD sent immediately → board echoes the code. + fn board_code(plugin: &StageAModulationPlugin) -> Option { + plugin.shared.state.lock().unwrap().board_code + } + + /// Connect checkbox → slider change → MOD sent by the device thread → + /// board echoes the code. No process_frame involved anywhere. #[test] - fn level_change_transfers_immediately_and_board_code_is_shown() { + fn level_change_transfers_without_frames() { let mut plugin = StageAModulationPlugin::default(); - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); - assert_eq!(plugin.firmware, "0.3.0-mock"); - - plugin - .set_setting("level", json!(1234)) - .expect("level accepted"); - assert!(plugin.dirty); - plugin.send_modulation(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.board_code == Some(1234) + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + assert_eq!( + plugin.shared.state.lock().unwrap().firmware, + "0.3.0-mock".to_owned() + ); + + plugin.set_setting("level", json!(1234)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1234) }); - assert!(!plugin.dirty); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - plugin.disconnect("test done"); + assert!(plugin.shared.state.lock().unwrap().last_error.is_none()); + + plugin.set_setting("connect", json!(false)).unwrap(); + assert!(!plugin.device_connected()); } /// The max cap bounds the slider, and lowering it re-sends a lower level. @@ -874,7 +899,6 @@ mod tests { plugin.set_setting("max_level", json!(500)).unwrap(); assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); - assert!(plugin.dirty, "the lowered level must be transferred"); let schema = plugin.settings_schema(); let level_item = schema.sections[0] @@ -888,28 +912,36 @@ mod tests { } } - /// Square drive with min threshold reaches the mock and starts at min. + /// Square drive with min threshold reaches the mock and starts at min; + /// slider to 0 drives the output to 0. #[test] fn square_with_min_threshold_round_trips() { let mut plugin = StageAModulationPlugin::default(); - plugin.connect(); - drain_until(&mut plugin, Duration::from_secs(2), |p| p.connected); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); plugin.set_setting("level", json!(2000)).unwrap(); - plugin.set_setting("mode", json!("SQUARE")).unwrap(); plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); plugin.set_setting("min_level", json!(500)).unwrap(); - plugin.send_modulation(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.board_code == Some(500) + plugin.set_setting("mode", json!("SQUARE")).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(500) }); - assert!(plugin.board_mod.contains("SQUARE 500..2000")); - - plugin.output_off(); - drain_until(&mut plugin, Duration::from_secs(2), |p| { - p.board_code == Some(0) + assert!(plugin + .shared + .state + .lock() + .unwrap() + .board_mod + .contains("SQUARE 500..2000")); + + plugin.set_setting("mode", json!("CONST")).unwrap(); + plugin.set_setting("level", json!(0)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(0) }); - plugin.disconnect("test done"); + plugin.set_setting("connect", json!(false)).unwrap(); } /// The host settings UI exchanges enum values as indices into the @@ -923,11 +955,11 @@ mod tests { .expect("index accepted"); assert_eq!(plugin.mode, Mode::Square); assert_eq!(plugin.get_setting("mode"), Some(json!(2))); - // Port: index 1 = "auto" (variants start with mock, auto). + // Port: index 1 = "mock" (variants start with auto, mock). plugin .set_setting("port", json!(1)) .expect("index accepted"); - assert_eq!(plugin.port_hint, "auto"); + assert_eq!(plugin.port_hint, "mock"); assert_eq!(plugin.get_setting("port"), Some(json!(1))); // Out-of-range indices are visible errors, not silent no-ops. assert!(plugin.set_setting("mode", json!(99)).is_err()); diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index d6f2928..7957f17 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -22,12 +22,11 @@ use std::thread::JoinHandle; use std::time::{Duration, Instant}; use augur_plugin_api::{ - export_plugin, EventStoreHandle, HostActionDescriptor, HostActionRequestQueue, HostActionScope, - HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, - HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, PluginFrame, Series1dLine, - Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, - StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, - TableValueType, CTX_INVESTIGATION_ACTION_REQUESTS, + export_plugin, EventStoreHandle, HostContext, HostDatasetDescriptor, HostDatasetKind, + HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, + PluginFrame, Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, }; use serde_json::{json, Value}; @@ -36,9 +35,6 @@ const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; const STATUS_DATASET_ID: &str = "stage-a-photodiode.status"; const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; -const ACTION_CONNECT: &str = "stage-a-photodiode.connect"; -const ACTION_DISCONNECT: &str = "stage-a-photodiode.disconnect"; - const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; /// Ring capacity: > 2.5 minutes at the firmware's 50 lines/s. @@ -226,14 +222,13 @@ pub struct StageAPhotodiodePlugin { reader: Option, shared: Arc>, generation: Arc, - effects_blocked_reason: Option, last_error: Option, // -- settings -- + connect_requested: bool, port_hint: String, mode: Mode, reference_volts: f64, window_s: f64, - consumed_action_ids: Vec, } impl Default for StageAPhotodiodePlugin { @@ -243,13 +238,12 @@ impl Default for StageAPhotodiodePlugin { reader: None, shared: Arc::new(Mutex::new(SharedState::default())), generation: Arc::new(AtomicU64::new(1)), - effects_blocked_reason: None, last_error: None, - port_hint: "mock".into(), + connect_requested: false, + port_hint: "auto".into(), mode: Mode::Raw, reference_volts: 3.3, window_s: 10.0, - consumed_action_ids: Vec::new(), } } } @@ -287,7 +281,10 @@ impl StageAPhotodiodePlugin { }; match Reader::spawn_serial(path, Arc::clone(&self.shared), Arc::clone(&self.generation)) { Ok(reader) => self.reader = Some(reader), - Err(err) => self.last_error = Some(err), + Err(err) => { + self.last_error = Some(err); + self.connect_requested = false; + } } self.generation.fetch_add(1, Ordering::Relaxed); } @@ -305,29 +302,6 @@ impl StageAPhotodiodePlugin { } } - fn consume_actions(&mut self, context: &HostContext<'_>) -> Vec { - let Ok(Some(queue)) = - context.get::(CTX_INVESTIGATION_ACTION_REQUESTS) - else { - return Vec::new(); - }; - let mut consumed = Vec::new(); - for request in queue.requests { - if self.consumed_action_ids.contains(&request.request_id) { - continue; - } - if !request.action_id.starts_with("stage-a-photodiode.") { - continue; - } - self.consumed_action_ids.push(request.request_id); - if self.consumed_action_ids.len() > 256 { - self.consumed_action_ids.remove(0); - } - consumed.push(request.action_id); - } - consumed - } - fn series_dataset(&self) -> Series1dV1 { let (points, y_label) = match self.shared.lock() { Ok(state) => { @@ -369,10 +343,10 @@ impl StageAPhotodiodePlugin { Ok(state) => (state.latest, state.error.clone()), Err(_) => (None, None), }; - let state = match (&self.effects_blocked_reason, self.connected()) { - (Some(reason), _) => format!("locked ({reason})"), - (None, false) => "disconnected".into(), - (None, true) => format!("reading ({})", self.port_hint), + let state = if self.connected() { + format!("reading ({})", self.port_hint) + } else { + "disconnected".into() }; let (code_text, value_text) = match latest { Some(sample) => ( @@ -547,6 +521,7 @@ impl Plugin for StageAPhotodiodePlugin { fn set_enabled(&mut self, enabled: bool) { self.enabled = enabled; if !enabled { + self.connect_requested = false; self.disconnect(); } } @@ -562,31 +537,12 @@ impl Plugin for StageAPhotodiodePlugin { &mut self, _frame: &PluginFrame<'_>, _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, + _context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - // The stream port is read-only, but device access still follows the - // same fail-closed gate as every stage-a plugin. - let execution = context.execution(); - if !execution.hardware_effects_allowed() { - self.effects_blocked_reason = Some(format!( - "hardware effects not allowed in {:?}", - execution.mode - )); - if self.reader.is_some() { - self.disconnect(); - } - return; - } - self.effects_blocked_reason = None; - - for action_id in self.consume_actions(context) { - match action_id.as_str() { - ACTION_CONNECT => self.connect(), - ACTION_DISCONNECT => self.disconnect(), - _ => {} - } - } + // Reading is settings-driven (connect checkbox) and works without + // camera frames; the stream port carries no commands, so no replay + // teardown is needed either. } fn settings_schema(&self) -> SettingsSchema { @@ -626,6 +582,16 @@ impl Plugin for StageAPhotodiodePlugin { default: port_default, }, }, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the stream port (read-only, no camera required).".into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, + }, SettingItem { key: "mode".into(), label: "Mode".into(), @@ -679,6 +645,7 @@ impl Plugin for StageAPhotodiodePlugin { .unwrap_or(0); Some(json!(index)) } + "connect" => Some(json!(self.connect_requested)), "mode" => { let index = Mode::VARIANTS .iter() @@ -698,6 +665,16 @@ impl Plugin for StageAPhotodiodePlugin { self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); Ok(()) } + "connect" => { + let requested = value.as_bool().ok_or("connect must be a boolean")?; + self.connect_requested = requested; + if requested { + self.connect(); + } else { + self.disconnect(); + } + Ok(()) + } "mode" => { let mode_names: Vec = Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); @@ -722,9 +699,6 @@ impl Plugin for StageAPhotodiodePlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - if let Some(reason) = &self.effects_blocked_reason { - entries.push(StatusEntry::Text(format!("Hardware locked: {reason}"))); - } let (latest, stream_error) = match self.shared.lock() { Ok(state) => (state.latest, state.error.clone()), Err(_) => (None, None), @@ -756,14 +730,6 @@ impl Plugin for StageAPhotodiodePlugin { } fn host_views(&self) -> HostViewRegistry { - let action = |id: &str, title: &str| HostActionDescriptor { - id: id.into(), - title: title.into(), - scope: HostActionScope::Dataset { - dataset_id: STATUS_DATASET_ID.into(), - }, - param_schema: None, - }; HostViewRegistry { datasets: vec![ HostDatasetDescriptor { @@ -799,10 +765,7 @@ impl Plugin for StageAPhotodiodePlugin { kind: HostViewKind::CompactTable, }, ], - actions: vec![ - action(ACTION_CONNECT, "Connect"), - action(ACTION_DISCONNECT, "Disconnect"), - ], + actions: Vec::new(), } } From 5f323c2012c1cb3db0f4abd5adac2c46384fa12d Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Thu, 16 Jul 2026 16:29:29 +0200 Subject: [PATCH 15/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20read=20the?= =?UTF-8?q?=20PDA1=20photodiode=20stream=20at=2020=20kSa/s=20with=20envelo?= =?UTF-8?q?pe=20and=20moving=20average?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track firmware 0.4.0 (ADR 003 in stage-a-controller): the stream port now carries PDA1 SamplesU16 frames at pd_stream_rate_hz instead of 50 Hz ASCII lines, raising the plot's data rate 400x. - parse with stage-a-io's FrameParser (new dep, default-features off: wire parser only); auto port probe detects sample frames - bounded raw ring (130 s / 4 M samples) keyed by device sample index; rate changes and index jumps restart the segment so index/rate stays a consistent time base across acquisition handovers - chart decimates the window into <= 1000 min/mean/max envelope buckets; short windows render raw samples; window down to 10 ms - moving-average indicator for the low-voltage regime: fixed sample window (default 4) or one full period of a user-set sync frequency (window = rate / f), making the mean phase-independent under modulation; overlay line + numeric readout in status views - status table gains rate, moving avg, and stream-integrity columns (device drops, CRC failures, resync bytes, segment restarts) - stage-a-io: gate the Duration import behind the hardware feature so default-features = false builds are warning-free Verified: cargo fmt, clippy -D warnings (photodiode + stage-a-io), 9 plugin tests + 29 stage-a-io tests green. --- docs/features/README.md | 2 +- docs/features/stage-a-photodiode.md | 45 +- plugins/stage-a-photodiode/Cargo.toml | 1 + plugins/stage-a-photodiode/plugin.toml | 2 +- plugins/stage-a-photodiode/src/lib.rs | 751 +++++++++++++++++++------ stage-a-io/src/transport.rs | 1 + 6 files changed, 624 insertions(+), 178 deletions(-) diff --git a/docs/features/README.md b/docs/features/README.md index eb5ffe4..16ed650 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -6,7 +6,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. - [Stage-A Modulation](./stage-a-modulation.md) — capped power slider + constant/sine/square laser-modulation drive on the command port, applied immediately. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the stream port: raw values or excitation power `I_exc = I_tot − I_pd`. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 916ce8c..ca688b4 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -1,15 +1,18 @@ # Stage-A Photodiode - **Crate:** `plugins/stage-a-photodiode` (`augur-plugin-stage-a-photodiode`) -- **Firmware:** `stage-a-controller` 0.3.0+ (`PDSTREAM`), Teensy **stream port** (second CDC port) -- **Status:** Active (2026-07-15) — replaces the readout half of `stage-a-monitor` +- **Firmware:** `stage-a-controller` 0.4.0+ (`PDSTREAM_PDA1`), Teensy **stream port** (second CDC port) +- **Status:** Active (2026-07-16) — replaces the readout half of `stage-a-monitor` ## What it is -A minimal live readout of the photodiode on **board SMA5 → Teensy pin 18 / A4**. The firmware -streams `PD code= n= t_ms=` lines at 50 Hz on its second USB serial port; a -background thread parses them into a bounded ring, and the plugin shows the newest value plus a -rolling chart (1–120 s window). +A live readout of the photodiode on **board SMA5 → Teensy pin 18 / A4**. Firmware 0.4.0 streams +PDA1 `SamplesU16` frames free-running at `pd_stream_rate_hz` (20 kSa/s default) on its second USB +serial port; a background thread parses them with `stage-a-io`'s `FrameParser` into a bounded raw +ring (up to 130 s / 4 M samples), and the plugin renders a rolling chart (10 ms – 120 s window) +plus the newest value. During a command-port acquisition the firmware mirrors the acquisition +blocks here — every rate change or sample-index jump restarts the ring as a new segment, so the +`index / rate` time base is always consistent. Two modes: @@ -18,18 +21,34 @@ Two modes: removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. +## Chart + +- The visible window is decimated into at most 1 000 buckets; when a bucket covers more than one + sample the chart shows the bucket **mean** plus a **min/max envelope**, so narrow modulation + peaks stay visible at any zoom. Windows short enough to fit raw samples render them directly. +- **Moving average** (for the low-voltage regime): a smoothed overlay line plus a numeric readout. + The window is either a fixed sample count (`avg_samples`, default 4; 1 = off) or — the right + tool for modulated signals — **one full period of a user-given frequency** + (`avg_sync_freq_hz`, e.g. the MOD drive frequency): window = `rate / f` samples, which makes + the mean independent of the modulation phase instead of riding the waveform. + ## Contract - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the - plugin is read-only by construction and needs no protocol library — it depends only on - `serialport` and parses one line format. + plugin is read-only by construction. It reuses `stage-a-io` (`default-features = false`) only + for the PDA1 wire parser — no client, worker, or transport. - **Frame-independent**: connecting is a checkbox setting; the reader thread and all views work with no camera attached (the host only calls `process_frame()` while frames flow). -- Garbage on the port (e.g. the binary command port picked by mistake) parses to nothing and is - bounded — it can neither grow memory nor produce fake values. -- `mock` port synthesizes a slow sine for hardware-free testing. +- Garbage on the port resynchronises at the next CRC-clean frame; skipped bytes and CRC failures + are counted and shown in the status table's integrity column together with the firmware's + cumulative drop counter and the segment-restart count. +- `mock` port synthesizes a noisy 5 Hz sine at 20 kSa/s in firmware-sized blocks for + hardware-free testing. ## Verification -`cargo test -p augur-plugin-stage-a-photodiode` — line parsing (including clamping and rejection), -excitation inversion against the reference, mock reader filling ring/series, ring bound. +`cargo test -p augur-plugin-stage-a-photodiode` — frame ingestion incl. segment restarts on index +jumps and rate changes, duration-bounded ring with aligned indexes, moving-average window +derivation from the sync frequency, newest-window average, envelope decimation bounds and +min ≤ mean ≤ max, raw rendering for short windows, excitation inversion, mock reader, settings +round-trips. diff --git a/plugins/stage-a-photodiode/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml index b426623..a838ebb 100644 --- a/plugins/stage-a-photodiode/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -13,3 +13,4 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true serialport.workspace = true +stage-a-io = { path = "../../stage-a-io", default-features = false } diff --git a/plugins/stage-a-photodiode/plugin.toml b/plugins/stage-a-photodiode/plugin.toml index 702a218..21b4bad 100644 --- a/plugins/stage-a-photodiode/plugin.toml +++ b/plugins/stage-a-photodiode/plugin.toml @@ -1,5 +1,5 @@ name = "Stage-A Photodiode" -version = "0.3.0" +version = "0.4.0" description = "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." domain = "stage-a" library = "augur_plugin_stage_a_photodiode" diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 7957f17..a966045 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -1,11 +1,13 @@ //! Stage-A photodiode readout. //! -//! Reads the free-running ASCII stream the `stage-a-controller` firmware -//! (0.3.0+, `USB_DUAL_SERIAL`) emits on its **second** USB serial port: -//! one `PD code= n= t_ms=` line every 20 ms from the -//! photodiode on board SMA5 → Teensy pin 18 / A4. The port carries no +//! Reads the free-running PDA1 binary frame stream the `stage-a-controller` +//! firmware (0.4.0+, `USB_DUAL_SERIAL`) emits on its **second** USB serial +//! port: `SamplesU16` frames at `pd_stream_rate_hz` (20 kSa/s default) from +//! the photodiode on board SMA5 → Teensy pin 18 / A4. The port carries no //! commands, so opening it is side-effect free; the command port is owned by -//! `stage-a-modulation`. +//! `stage-a-modulation`. While a command-port acquisition runs the firmware +//! mirrors its blocks here (flag 0x0001) — every rate change or sample-index +//! jump is treated as a segment restart. //! //! Two display modes: //! - **RAW**: the ADC code and its voltage (`V = code · 3.3 / 4095`); @@ -13,6 +15,11 @@ //! path and sees the light removed from the beam, `I_pd = I_tot − I_exc`. //! Given the user-set reference `I_tot` (in photodiode volts), the plugin //! shows `I_exc = I_tot − V_pd`. +//! +//! The chart decimates the visible window into min/mean/max envelope buckets +//! and overlays a moving average whose window is either a fixed sample count +//! or — for modulated signals — one full period of a user-given frequency, +//! which makes the mean independent of the modulation phase. use std::collections::VecDeque; use std::io::Read; @@ -29,6 +36,7 @@ use augur_plugin_api::{ TableSchema, TableValueType, }; use serde_json::{json, Value}; +use stage_a_io::{FrameParser, ParseEvent}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; @@ -37,8 +45,17 @@ const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; -/// Ring capacity: > 2.5 minutes at the firmware's 50 lines/s. -const RING_CAPACITY: usize = 8_192; +/// Longest raw history kept, in seconds of samples at the active stream rate. +const RING_SECONDS: f64 = 130.0; +/// Absolute sample cap guarding against absurd advertised rates (8 MiB of +/// codes at most). +const RING_MAX_SAMPLES: usize = 4_000_000; +/// Envelope buckets per rendered chart line; keeps the plot payload bounded +/// no matter how many raw samples the window covers. +const MAX_PLOT_BUCKETS: usize = 1_000; +/// The firmware's default stream rate; the mock mirrors it. +const MOCK_RATE_HZ: u32 = 20_000; +const MOCK_BLOCK_SAMPLES: usize = 256; fn code_to_volts(code: f64) -> f64 { code * ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE @@ -65,45 +82,56 @@ impl Mode { } } -#[derive(Debug, Clone, Copy)] -struct PdSample { - t_ms: u64, - code: f64, -} - -/// Parses one firmware stream line: `PD code= n= t_ms=`. -fn parse_pd_line(line: &str) -> Option { - let rest = line.trim().strip_prefix("PD ")?; - let mut code = None; - let mut t_ms = None; - for token in rest.split_ascii_whitespace() { - let (key, value) = token.split_once('=')?; - match key { - "code" => code = value.parse::().ok(), - "t_ms" => t_ms = value.parse::().ok(), - "n" => {} - _ => return None, - } - } - Some(PdSample { - t_ms: t_ms?, - code: code?.clamp(0.0, ADC_MAX_CODE), - }) -} - #[derive(Default)] struct SharedState { - samples: VecDeque, - latest: Option, + /// Sample rate of the current segment (from the frame headers). + rate_hz: u32, + /// Device sample index of `samples.front()` within the current segment. + ring_first_index: u64, + samples: VecDeque, + latest: Option, + /// Cumulative firmware-side drop counter (latest header value). + device_dropped: u32, + crc_failures: u64, + resync_bytes: u64, + /// Segment restarts observed (rate changes, index jumps, reconnects). + segments: u64, error: Option, } impl SharedState { - fn push(&mut self, sample: PdSample) { - self.latest = Some(sample); - self.samples.push_back(sample); - while self.samples.len() > RING_CAPACITY { - self.samples.pop_front(); + fn ring_capacity(rate_hz: u32) -> usize { + ((f64::from(rate_hz.max(1)) * RING_SECONDS) as usize).min(RING_MAX_SAMPLES) + } + + /// Ingests one `SamplesU16` frame. Any discontinuity — rate change, + /// sample-index jump (drops, acquisition handover), reconnect — restarts + /// the ring: within a segment `index / rate` is a consistent time base. + fn ingest(&mut self, first_index: u64, rate_hz: u32, device_dropped: u32, codes: &[u16]) { + if codes.is_empty() { + return; + } + let expected = self.ring_first_index + self.samples.len() as u64; + let continuous = + !self.samples.is_empty() && rate_hz == self.rate_hz && first_index == expected; + if !continuous { + if !self.samples.is_empty() { + self.segments += 1; + } + self.samples.clear(); + self.ring_first_index = first_index; + self.rate_hz = rate_hz; + } + self.samples.extend(codes.iter().copied()); + self.latest = codes.last().copied(); + self.device_dropped = device_dropped; + let excess = self + .samples + .len() + .saturating_sub(Self::ring_capacity(rate_hz)); + if excess > 0 { + self.samples.drain(..excess); + self.ring_first_index += excess as u64; } } } @@ -128,7 +156,7 @@ impl Reader { let thread_stop = Arc::clone(&stop); let join = std::thread::Builder::new() .name("stage-a-photodiode".into()) - .spawn(move || read_lines(port, &shared, &generation, &thread_stop)) + .spawn(move || read_frames(port, &shared, &generation, &thread_stop)) .expect("spawning the photodiode reader thread must succeed"); Ok(Self { stop, @@ -136,7 +164,8 @@ impl Reader { }) } - /// Hardware-free source: synthesizes a slow sine around 1 V at 50 Hz. + /// Hardware-free source: synthesizes a noisy 5 Hz sine around 1 V in + /// firmware-sized blocks at the firmware's default stream rate. fn spawn_mock(shared: Arc>, generation: Arc) -> Self { let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); @@ -144,18 +173,24 @@ impl Reader { .name("stage-a-photodiode-mock".into()) .spawn(move || { let start = Instant::now(); + let mut next_index: u64 = 0; while !thread_stop.load(Ordering::Relaxed) { - let t = start.elapsed().as_secs_f64(); - let volts = 1.0 + 0.5 * (2.0 * std::f64::consts::PI * 0.2 * t).sin(); - let sample = PdSample { - t_ms: (t * 1_000.0) as u64, - code: volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS, - }; - if let Ok(mut state) = shared.lock() { - state.push(sample); + let target = (start.elapsed().as_secs_f64() * f64::from(MOCK_RATE_HZ)) as u64; + let mut produced = false; + while next_index + MOCK_BLOCK_SAMPLES as u64 <= target { + let codes: Vec = (0..MOCK_BLOCK_SAMPLES) + .map(|i| mock_code(next_index + i as u64)) + .collect(); + if let Ok(mut state) = shared.lock() { + state.ingest(next_index, MOCK_RATE_HZ, 0, &codes); + } + next_index += MOCK_BLOCK_SAMPLES as u64; + produced = true; + } + if produced { + generation.fetch_add(1, Ordering::Relaxed); } - generation.fetch_add(1, Ordering::Relaxed); - std::thread::sleep(Duration::from_millis(20)); + std::thread::sleep(Duration::from_millis(5)); } }) .expect("spawning the mock photodiode thread must succeed"); @@ -166,6 +201,17 @@ impl Reader { } } +/// Deterministic mock sample: 1 V ± 0.5 V sine at 5 Hz plus ~20 mV of hash +/// noise, so the moving-average indicator has something to smooth. +fn mock_code(index: u64) -> u16 { + let t = index as f64 / f64::from(MOCK_RATE_HZ); + let mut hash = index.wrapping_mul(0x9E37_79B9_7F4A_7C15); + hash ^= hash >> 33; + let noise = (hash as f64 / u64::MAX as f64) - 0.5; + let volts = 1.0 + 0.5 * (2.0 * std::f64::consts::PI * 5.0 * t).sin() + 0.04 * noise; + (volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS).clamp(0.0, ADC_MAX_CODE) as u16 +} + impl Drop for Reader { fn drop(&mut self) { self.stop.store(true, Ordering::Relaxed); @@ -175,14 +221,14 @@ impl Drop for Reader { } } -fn read_lines( +fn read_frames( mut port: Box, shared: &Mutex, generation: &AtomicU64, stop: &AtomicBool, ) { - let mut line_buffer: Vec = Vec::with_capacity(256); - let mut buf = [0_u8; 512]; + let mut parser = FrameParser::default(); + let mut buf = [0_u8; 4_096]; while !stop.load(Ordering::Relaxed) { let read = match port.read(&mut buf) { Ok(0) => continue, @@ -197,23 +243,39 @@ fn read_lines( return; } }; - line_buffer.extend_from_slice(&buf[..read]); - // Never let garbage (e.g. the wrong, binary port) grow the buffer. - if line_buffer.len() > 4_096 { - line_buffer.clear(); - } - while let Some(pos) = line_buffer.iter().position(|&b| b == b'\n') { - let line: Vec = line_buffer.drain(..=pos).collect(); - let Ok(text) = std::str::from_utf8(&line) else { - continue; - }; - if let Some(sample) = parse_pd_line(text) { - if let Ok(mut state) = shared.lock() { - state.push(sample); + parser.extend(&buf[..read]); + let mut changed = false; + while let Some(event) = parser.next_event() { + match event { + ParseEvent::Frame(frame) => { + let Some(codes) = frame.samples() else { + continue; // Control/summary frames are not expected here. + }; + if let Ok(mut state) = shared.lock() { + state.ingest( + frame.header.first_sample_index, + frame.header.sample_rate_hz, + frame.header.dropped_samples, + &codes, + ); + } + changed = true; + } + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + if let Ok(mut state) = shared.lock() { + state.resync_bytes += skipped_bytes as u64; + state.crc_failures += crc_failures as u64; + } + changed = true; } - generation.fetch_add(1, Ordering::Relaxed); } } + if changed { + generation.fetch_add(1, Ordering::Relaxed); + } } } @@ -229,6 +291,8 @@ pub struct StageAPhotodiodePlugin { mode: Mode, reference_volts: f64, window_s: f64, + avg_samples: usize, + avg_sync_freq_hz: f64, } impl Default for StageAPhotodiodePlugin { @@ -244,6 +308,8 @@ impl Default for StageAPhotodiodePlugin { mode: Mode::Raw, reference_volts: 3.3, window_s: 10.0, + avg_samples: 4, + avg_sync_freq_hz: 0.0, } } } @@ -302,59 +368,211 @@ impl StageAPhotodiodePlugin { } } + /// Moving-average window in samples: either the fixed sample count or, + /// when a sync frequency is set, one full period of that frequency — + /// which makes the mean independent of the modulation phase. + fn avg_window_samples(&self, rate_hz: u32) -> usize { + if self.avg_sync_freq_hz > 0.0 && rate_hz > 0 { + (f64::from(rate_hz) / self.avg_sync_freq_hz) + .round() + .max(1.0) as usize + } else { + self.avg_samples.max(1) + } + } + + /// Mean of the newest `avg_window_samples` codes (fewer while filling). + fn current_average_code(&self, state: &SharedState) -> Option { + if state.samples.is_empty() { + return None; + } + let window = self + .avg_window_samples(state.rate_hz) + .min(state.samples.len()); + let start = state.samples.len() - window; + let sum: u64 = state.samples.range(start..).map(|&c| u64::from(c)).sum(); + Some(sum as f64 / window as f64) + } + fn series_dataset(&self) -> Series1dV1 { - let (points, y_label) = match self.shared.lock() { - Ok(state) => { - let latest_ms = state.latest.map_or(0, |s| s.t_ms); - let window_ms = (self.window_s.max(0.5) * 1_000.0) as u64; - let cutoff = latest_ms.saturating_sub(window_ms); - let points: Vec = state - .samples - .iter() - .filter(|s| s.t_ms >= cutoff) - .map(|s| Series1dPoint { - x: (s.t_ms as f64 - latest_ms as f64) / 1_000.0, - y: self.display_volts(s.code), - }) - .collect(); - let label = match self.mode { - Mode::Raw => "photodiode [V]", - Mode::Excitation => "excitation I_tot − I_pd [V]", - }; - (points, label) - } - Err(_) => (Vec::new(), "photodiode [V]"), + let y_label = match self.mode { + Mode::Raw => "photodiode [V]", + Mode::Excitation => "excitation I_tot − I_pd [V]", }; - Series1dV1 { + let trace_name = match self.mode { + Mode::Raw => "photodiode", + Mode::Excitation => "excitation", + }; + let empty = |label: &str| Series1dV1 { x_label: "time before now [s]".into(), - y_label: y_label.into(), + y_label: label.into(), lines: vec![Series1dLine { - name: match self.mode { - Mode::Raw => "photodiode".into(), - Mode::Excitation => "excitation".into(), - }, - points, + name: trace_name.into(), + points: Vec::new(), }], + }; + let Ok(state) = self.shared.lock() else { + return empty(y_label); + }; + let total = state.samples.len(); + if total == 0 || state.rate_hz == 0 { + return empty(y_label); + } + let rate = f64::from(state.rate_hz); + + let visible = ((self.window_s.max(0.001) * rate) as usize) + .max(2) + .min(total); + let start = total - visible; + let latest_x_index = state.ring_first_index + total as u64 - 1; + let bucket_len = visible.div_ceil(MAX_PLOT_BUCKETS).max(1); + let decimating = bucket_len > 1; + + let avg_window = self.avg_window_samples(state.rate_hz); + let avg_enabled = avg_window > 1; + // Prime the running sum with up to `avg_window − 1` samples that + // precede the visible slice, so the average is correct from the + // first visible point on. + let prime_start = start.saturating_sub(avg_window - 1); + let mut avg_sum: u64 = 0; + let mut avg_count: usize = 0; + for &code in state.samples.range(prime_start..start) { + avg_sum += u64::from(code); + avg_count += 1; + } + + let mut mean_points = Vec::with_capacity(MAX_PLOT_BUCKETS + 1); + let mut min_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); + let mut max_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); + let mut avg_points = Vec::with_capacity(if avg_enabled { MAX_PLOT_BUCKETS + 1 } else { 0 }); + + let mut bucket_min = u16::MAX; + let mut bucket_max = u16::MIN; + let mut bucket_sum: u64 = 0; + let mut bucket_n: usize = 0; + for (offset, &code) in state.samples.range(start..).enumerate() { + let i = start + offset; + bucket_min = bucket_min.min(code); + bucket_max = bucket_max.max(code); + bucket_sum += u64::from(code); + bucket_n += 1; + if avg_enabled { + avg_sum += u64::from(code); + avg_count += 1; + if avg_count > avg_window { + avg_sum -= u64::from(state.samples[i - avg_window]); + avg_count -= 1; + } + } + if bucket_n == bucket_len || i == total - 1 { + let x = (state.ring_first_index + i as u64) as f64 / rate + - latest_x_index as f64 / rate; + mean_points.push(Series1dPoint { + x, + y: self.display_volts(bucket_sum as f64 / bucket_n as f64), + }); + if decimating { + // EXCITATION inverts the axis, so min/max swap roles. + let (low, high) = ( + self.display_volts(f64::from(bucket_min)), + self.display_volts(f64::from(bucket_max)), + ); + min_points.push(Series1dPoint { + x, + y: low.min(high), + }); + max_points.push(Series1dPoint { + x, + y: low.max(high), + }); + } + if avg_enabled { + avg_points.push(Series1dPoint { + x, + y: self.display_volts(avg_sum as f64 / avg_count as f64), + }); + } + bucket_min = u16::MAX; + bucket_max = u16::MIN; + bucket_sum = 0; + bucket_n = 0; + } + } + + let mut lines = vec![Series1dLine { + name: trace_name.into(), + points: mean_points, + }]; + if decimating { + lines.push(Series1dLine { + name: "min".into(), + points: min_points, + }); + lines.push(Series1dLine { + name: "max".into(), + points: max_points, + }); + } + if avg_enabled { + lines.push(Series1dLine { + name: format!("avg ({avg_window} spl)"), + points: avg_points, + }); + } + Series1dV1 { + x_label: "time before now [s]".into(), + y_label: y_label.into(), + lines, } } fn status_dataset(&self) -> TableDatasetV1 { - let (latest, stream_error) = match self.shared.lock() { - Ok(state) => (state.latest, state.error.clone()), - Err(_) => (None, None), + let (latest, rate_hz, average, integrity, stream_error) = match self.shared.lock() { + Ok(state) => ( + state.latest, + state.rate_hz, + self.current_average_code(&state), + format!( + "drops={} crc={} resync={} segments={}", + state.device_dropped, state.crc_failures, state.resync_bytes, state.segments + ), + state.error.clone(), + ), + Err(_) => (None, 0, None, String::new(), None), }; - let state = if self.connected() { + let state_text = if self.connected() { format!("reading ({})", self.port_hint) } else { "disconnected".into() }; + let rate_text = if rate_hz > 0 { + format!("{rate_hz} Sa/s") + } else { + "—".into() + }; let (code_text, value_text) = match latest { Some(sample) => ( - format!("{:.1}", sample.code), - format!("{:.4} V", self.display_volts(sample.code)), + format!("{sample}"), + format!("{:.4} V", self.display_volts(f64::from(sample))), ), None => ("—".into(), "—".into()), }; + let avg_text = match average { + Some(code) => { + let window = self.avg_window_samples(rate_hz); + format!( + "{:.4} V ({} spl ≈ {:.2} ms)", + self.display_volts(code), + window, + if rate_hz > 0 { + window as f64 * 1_000.0 / f64::from(rate_hz) + } else { + 0.0 + } + ) + } + None => "—".into(), + }; let error = stream_error .or_else(|| self.last_error.clone()) .unwrap_or_default(); @@ -364,10 +582,13 @@ impl StageAPhotodiodePlugin { }; TableDatasetV1 { columns: vec![ - text_column("state", state), + text_column("state", state_text), text_column("mode", self.mode.name().to_owned()), + text_column("rate", rate_text), text_column("code", code_text), text_column("value", value_text), + text_column("avg", avg_text), + text_column("integrity", integrity), text_column("error", error), ], } @@ -383,8 +604,11 @@ impl StageAPhotodiodePlugin { columns: vec![ column("state", "State"), column("mode", "Mode"), + column("rate", "Rate"), column("code", "ADC code"), column("value", "Value"), + column("avg", "Moving avg"), + column("integrity", "Integrity"), column("error", "Last error"), ], ..TableSchema::default() @@ -405,8 +629,9 @@ fn serial_ports() -> Vec { .unwrap_or_default() } -/// Finds the Teensy stream port: the dual-serial firmware free-runs `PD` -/// lines on exactly one of the enumerated ports, so listen briefly on each. +/// Finds the Teensy stream port: the dual-serial firmware free-runs PDA1 +/// `SamplesU16` frames on exactly one of the enumerated ports, so listen +/// briefly on each. fn resolve_auto_port() -> Result { let candidates = serial_ports(); if candidates.is_empty() { @@ -418,12 +643,14 @@ fn resolve_auto_port() -> Result { } } Err(format!( - "no port streamed PD lines within 500 ms (tried {})", + "no port streamed PDA1 sample frames within 500 ms (tried {})", candidates.join(", ") )) } -/// True when `path` produces a parsable `PD …` line within the probe window. +/// True when `path` produces a CRC-clean `SamplesU16` frame within the probe +/// window. The command port emits frames too, but only control replies and +/// acquisition data — unsolicited sample frames identify the stream port. fn probe_pd_stream(path: &str) -> bool { let Ok(mut port) = serialport::new(path, 115_200) .timeout(Duration::from_millis(100)) @@ -432,20 +659,18 @@ fn probe_pd_stream(path: &str) -> bool { return false; }; let deadline = Instant::now() + Duration::from_millis(500); - let mut collected: Vec = Vec::new(); - let mut buf = [0_u8; 512]; + let mut parser = FrameParser::default(); + let mut buf = [0_u8; 4_096]; while Instant::now() < deadline { match port.read(&mut buf) { Ok(read) if read > 0 => { - collected.extend_from_slice(&buf[..read]); - if String::from_utf8_lossy(&collected) - .lines() - .any(|line| parse_pd_line(line).is_some()) - { - return true; - } - if collected.len() > 8_192 { - collected.drain(..4_096); + parser.extend(&buf[..read]); + while let Some(event) = parser.next_event() { + if let ParseEvent::Frame(frame) = event { + if frame.samples().is_some() { + return true; + } + } } } Ok(_) => {} @@ -511,7 +736,7 @@ impl Plugin for StageAPhotodiodePlugin { } fn description(&self) -> &'static str { - "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot − I_pd with a user-set reference." + "Live photodiode readout (SMA5/pin 18/A4) from the Teensy PDA1 stream port at the full stream rate: raw values or excitation power I_exc = I_tot − I_pd with a user-set reference." } fn enabled(&self) -> bool { @@ -561,9 +786,10 @@ impl Plugin for StageAPhotodiodePlugin { sections: vec![SettingsSection { label: "Photodiode readout".into(), description: Some( - "Reads the free-running PD stream on the Teensy's SECOND serial port. \ - EXCITATION shows I_exc = I_tot − I_pd: the diode sits behind the PBS and \ - sees the light removed from the excitation beam." + "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ + port (firmware 0.4.0+, 20 kSa/s default). EXCITATION shows \ + I_exc = I_tot − I_pd: the diode sits behind the PBS and sees the light \ + removed from the excitation beam." .into(), ), default_open: true, @@ -573,8 +799,8 @@ impl Plugin for StageAPhotodiodePlugin { label: "Port".into(), tooltip: Some( "auto (recommended) listens on the attached usbmodem ports and \ - picks the one streaming PD lines — the Teensy stream port; \ - mock = synthetic data" + picks the one streaming PDA1 sample frames — the Teensy stream \ + port; mock = synthetic data" .into(), ), kind: SettingKind::Enum { @@ -621,14 +847,48 @@ impl Plugin for StageAPhotodiodePlugin { SettingItem { key: "window_s".into(), label: "Chart window".into(), - tooltip: Some("Seconds of history shown in the live chart".into()), + tooltip: Some( + "Seconds of history shown in the live chart. Short windows \ + (≤ 50 ms) resolve individual modulation cycles at 20 kSa/s." + .into(), + ), kind: SettingKind::F64Drag { - min: 1.0, + min: 0.01, max: 120.0, - speed: 1.0, + speed: 0.05, default: self.window_s, }, }, + SettingItem { + key: "avg_samples".into(), + label: "Average window".into(), + tooltip: Some( + "Moving-average window in samples (1 = off). Ignored while \ + 'Average sync frequency' is set." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 1_000_000, + default: self.avg_samples as i64, + }, + }, + SettingItem { + key: "avg_sync_freq_hz".into(), + label: "Average sync frequency".into(), + tooltip: Some( + "0 = off. When set to the modulation frequency (Hz), the moving \ + average spans exactly one full period (window = rate / f), so the \ + mean level no longer depends on the modulation phase." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 100_000.0, + speed: 1.0, + default: self.avg_sync_freq_hz, + }, + }, ], }], } @@ -655,6 +915,8 @@ impl Plugin for StageAPhotodiodePlugin { } "reference_volts" => Some(json!(self.reference_volts)), "window_s" => Some(json!(self.window_s)), + "avg_samples" => Some(json!(self.avg_samples)), + "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), _ => None, } } @@ -690,7 +952,17 @@ impl Plugin for StageAPhotodiodePlugin { } "window_s" => { let seconds = value.as_f64().ok_or("window_s must be a number")?; - self.window_s = seconds.clamp(1.0, 120.0); + self.window_s = seconds.clamp(0.01, 120.0); + Ok(()) + } + "avg_samples" => { + let samples = value.as_i64().ok_or("avg_samples must be an integer")?; + self.avg_samples = samples.clamp(1, 1_000_000) as usize; + Ok(()) + } + "avg_sync_freq_hz" => { + let freq = value.as_f64().ok_or("avg_sync_freq_hz must be a number")?; + self.avg_sync_freq_hz = freq.clamp(0.0, 100_000.0); Ok(()) } _ => Err(format!("unknown setting: {key}")), @@ -699,30 +971,47 @@ impl Plugin for StageAPhotodiodePlugin { fn status_entries(&self) -> Vec { let mut entries = Vec::new(); - let (latest, stream_error) = match self.shared.lock() { - Ok(state) => (state.latest, state.error.clone()), - Err(_) => (None, None), + let (latest, rate_hz, average, stream_error) = match self.shared.lock() { + Ok(state) => ( + state.latest, + state.rate_hz, + self.current_average_code(&state), + state.error.clone(), + ), + Err(_) => (None, 0, None, None), }; entries.push(StatusEntry::Text(if self.connected() { - format!("Photodiode: reading ({})", self.port_hint) + if rate_hz > 0 { + format!("Photodiode: reading ({}) @ {rate_hz} Sa/s", self.port_hint) + } else { + format!("Photodiode: reading ({})", self.port_hint) + } } else { "Photodiode: disconnected".into() })); if let Some(sample) = latest { match self.mode { Mode::Raw => entries.push(StatusEntry::Text(format!( - "PD: code={:.1} ({:.4} V)", - sample.code, - code_to_volts(sample.code) + "PD: code={sample} ({:.4} V)", + code_to_volts(f64::from(sample)) ))), Mode::Excitation => entries.push(StatusEntry::Text(format!( "Excitation: {:.4} V (I_tot={:.3} V, PD={:.4} V)", - self.display_volts(sample.code), + self.display_volts(f64::from(sample)), self.reference_volts, - code_to_volts(sample.code) + code_to_volts(f64::from(sample)) ))), } } + if let Some(average) = average { + let window = self.avg_window_samples(rate_hz); + if window > 1 { + entries.push(StatusEntry::Text(format!( + "Avg ({window} spl): {:.4} V", + self.display_volts(average) + ))); + } + } if let Some(error) = stream_error.or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } @@ -790,19 +1079,164 @@ export_plugin!(StageAPhotodiodePlugin); #[cfg(test)] mod tests { use super::*; + use stage_a_io::{Frame, FrameHeader, FrameType}; + + fn sample_frame(sequence: u32, first_index: u64, rate_hz: u32, codes: &[u16]) -> Vec { + let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); + Frame::build( + FrameHeader { + version: stage_a_io::wire::PROTOCOL_VERSION, + frame_type: FrameType::SamplesU16, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: first_index, + sample_rate_hz: rate_hz, + dropped_samples: 0, + crc32: 0, + }, + payload, + ) + .to_bytes() + } + + fn ingest_bytes(state: &mut SharedState, bytes: &[u8]) { + let mut parser = FrameParser::default(); + parser.extend(bytes); + while let Some(event) = parser.next_event() { + match event { + ParseEvent::Frame(frame) => { + let codes = frame.samples().expect("sample frame"); + state.ingest( + frame.header.first_sample_index, + frame.header.sample_rate_hz, + frame.header.dropped_samples, + &codes, + ); + } + ParseEvent::Corruption { .. } => panic!("clean test stream"), + } + } + } + + #[test] + fn ingests_contiguous_frames_and_restarts_on_gaps() { + let mut state = SharedState::default(); + ingest_bytes(&mut state, &sample_frame(0, 0, 20_000, &[1, 2, 3, 4])); + ingest_bytes(&mut state, &sample_frame(1, 4, 20_000, &[5, 6])); + assert_eq!(state.samples.len(), 6); + assert_eq!(state.ring_first_index, 0); + assert_eq!(state.segments, 0); + assert_eq!(state.latest, Some(6)); + + // A sample-index jump (dropped block, acquisition handover) restarts + // the segment instead of silently misaligning the time base. + ingest_bytes(&mut state, &sample_frame(2, 100, 20_000, &[7, 8])); + assert_eq!(state.samples.len(), 2); + assert_eq!(state.ring_first_index, 100); + assert_eq!(state.segments, 1); + + // So does a rate change (mirrored acquisition at another rate). + ingest_bytes(&mut state, &sample_frame(3, 102, 50_000, &[9])); + assert_eq!(state.samples.len(), 1); + assert_eq!(state.rate_hz, 50_000); + assert_eq!(state.segments, 2); + } #[test] - fn parses_firmware_stream_lines() { - let sample = parse_pd_line("PD code=1042.3 n=16 t_ms=123456\n").expect("valid line"); - assert!((sample.code - 1042.3).abs() < 1e-9); - assert_eq!(sample.t_ms, 123_456); + fn ring_is_bounded_by_duration() { + let mut state = SharedState::default(); + let rate = 1_000; // capacity = 130_000 samples + let cap = SharedState::ring_capacity(rate); + let block: Vec = (0..1_000).map(|i| (i % 4_096) as u16).collect(); + let mut index = 0_u64; + for _ in 0..(cap / block.len() + 5) { + state.ingest(index, rate, 0, &block); + index += block.len() as u64; + } + assert_eq!(state.samples.len(), cap); + assert_eq!( + state.ring_first_index + state.samples.len() as u64, + index, + "eviction keeps indexes aligned" + ); + assert_eq!(state.segments, 0, "eviction is not a discontinuity"); + } - assert!(parse_pd_line("garbage").is_none()); - assert!(parse_pd_line("PD code=abc n=16 t_ms=1").is_none()); - assert!(parse_pd_line("PD code=10 n=16").is_none(), "t_ms required"); - // Codes are clamped into the 12-bit range. - let clamped = parse_pd_line("PD code=9999 n=1 t_ms=5").expect("parses"); - assert_eq!(clamped.code, ADC_MAX_CODE); + #[test] + fn moving_average_window_follows_the_sync_frequency() { + let mut plugin = StageAPhotodiodePlugin::default(); + assert_eq!(plugin.avg_window_samples(20_000), 4, "sample default"); + plugin + .set_setting("avg_samples", json!(16)) + .expect("valid setting"); + assert_eq!(plugin.avg_window_samples(20_000), 16); + // One full period of a 2 kHz modulation at 20 kSa/s = 10 samples. + plugin + .set_setting("avg_sync_freq_hz", json!(2_000.0)) + .expect("valid setting"); + assert_eq!(plugin.avg_window_samples(20_000), 10); + // Faster than the sample rate clamps to a single sample. + plugin + .set_setting("avg_sync_freq_hz", json!(50_000.0)) + .expect("valid setting"); + assert_eq!(plugin.avg_window_samples(20_000), 1); + } + + #[test] + fn current_average_uses_the_newest_window() { + let plugin = StageAPhotodiodePlugin::default(); // window = 4 samples + let mut state = SharedState::default(); + state.ingest(0, 20_000, 0, &[0, 0, 0, 0, 100, 200, 300, 400]); + let average = plugin.current_average_code(&state).expect("has samples"); + assert!((average - 250.0).abs() < 1e-9); + } + + #[test] + fn series_dataset_decimates_with_envelope_and_average() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("window_s", json!(120.0)).unwrap(); + plugin.set_setting("avg_samples", json!(50)).unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + let codes: Vec = (0..40_000_u32).map(|i| (i % 4_000) as u16).collect(); + state.ingest(0, 20_000, 0, &codes); + } + let series = plugin.series_dataset(); + let names: Vec<&str> = series.lines.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["photodiode", "min", "max", "avg (50 spl)"]); + for line in &series.lines { + assert!( + line.points.len() <= MAX_PLOT_BUCKETS + 1, + "{} has {} points", + line.name, + line.points.len() + ); + assert!(!line.points.is_empty()); + } + // min ≤ mean ≤ max, and x is "seconds before now" ending at 0. + let (mean, min, max) = (&series.lines[0], &series.lines[1], &series.lines[2]); + for ((m, lo), hi) in mean.points.iter().zip(&min.points).zip(&max.points) { + assert!(lo.y <= m.y + 1e-9 && m.y <= hi.y + 1e-9); + } + let last_x = mean.points.last().unwrap().x; + assert!(last_x.abs() < 1e-9, "trace ends at now, got {last_x}"); + } + + #[test] + fn short_windows_render_raw_samples_without_envelope() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("window_s", json!(0.01)).unwrap(); // 200 samples at 20 kSa/s + plugin.set_setting("avg_samples", json!(1)).unwrap(); // average off + { + let mut state = plugin.shared.lock().unwrap(); + let codes: Vec = (0..1_000_u32).map(|i| (i % 4_000) as u16).collect(); + state.ingest(0, 20_000, 0, &codes); + } + let series = plugin.series_dataset(); + let names: Vec<&str> = series.lines.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["photodiode"], "no envelope, no average"); + assert_eq!(series.lines[0].points.len(), 200); } #[test] @@ -820,12 +1254,15 @@ mod tests { #[test] fn mock_reader_fills_the_ring_and_series() { - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = StageAPhotodiodePlugin { + port_hint: "mock".into(), + ..Default::default() + }; plugin.connect(); let deadline = Instant::now() + Duration::from_secs(2); loop { let count = plugin.shared.lock().unwrap().samples.len(); - if count >= 5 { + if count >= MOCK_BLOCK_SAMPLES { break; } assert!(Instant::now() < deadline, "mock reader produced no data"); @@ -833,6 +1270,7 @@ mod tests { } let series = plugin.series_dataset(); assert!(!series.lines[0].points.is_empty()); + assert_eq!(plugin.shared.lock().unwrap().rate_hz, MOCK_RATE_HZ); let generation = plugin.generation.load(Ordering::Relaxed); assert!(generation > 1); plugin.disconnect(); @@ -862,17 +1300,4 @@ mod tests { .expect("name accepted"); assert_eq!(plugin.mode, Mode::Raw); } - - #[test] - fn ring_is_bounded() { - let mut state = SharedState::default(); - for i in 0..(RING_CAPACITY + 100) { - state.push(PdSample { - t_ms: i as u64, - code: 1.0, - }); - } - assert_eq!(state.samples.len(), RING_CAPACITY); - assert_eq!(state.latest.unwrap().t_ms, (RING_CAPACITY + 99) as u64); - } } diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index f7e677a..e9c9377 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -6,6 +6,7 @@ use std::io; use std::sync::{Arc, Mutex}; +#[cfg(feature = "hardware")] use std::time::Duration; pub trait Transport: Send { From 25535811ec79cd7426a800cfa31194c2aa0c781d Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 16:52:19 +0200 Subject: [PATCH 16/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20monito?= =?UTF-8?q?r-cache=20snapshots=20and=20disk=20recording=20to=20the=20photo?= =?UTF-8?q?diode=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two save modes behind one Data settings section: - monitor cache: the raw ring is now cache_s seconds long (default 20 s, 1-130 s) and a Save-cache-snapshot button dumps it as pd_cache_.csv (sample_index, t_s on the device clock, raw code, raw volts) plus a JSON sidecar carrying rate, integrity counters, display mode, and reference so derived quantities stay reproducible - recording: a Record toggle tees every incoming SamplesU16 frame verbatim to pd_rec_.pdq via stage-a-io's PdqWriter from the reader thread; stopping (or disabling the plugin) finalizes the file and writes a sidecar with per-recording integrity deltas and validity; the mock synthesizes identical wire frames so recordings parse the same without hardware - data_dir uses the new Path setting kind; record/save failures surface through status entries like connect errors; status shows a live REC indicator with recorded seconds Verified: cargo fmt, clippy -D warnings, 13 plugin tests green. --- plugins/stage-a-photodiode/src/lib.rs | 780 ++++++++++++++++++++++---- 1 file changed, 676 insertions(+), 104 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index a966045..6f91025 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -22,12 +22,15 @@ //! which makes the mean independent of the modulation phase. use std::collections::VecDeque; -use std::io::Read; +use std::fs::File; +use std::io::{BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; +use augur_plugin_api::PathDialogKind; use augur_plugin_api::{ export_plugin, EventStoreHandle, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, @@ -36,7 +39,7 @@ use augur_plugin_api::{ TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{FrameParser, ParseEvent}; +use stage_a_io::{FrameParser, ParseEvent, PdqWriter, StreamIntegrity}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; @@ -45,8 +48,10 @@ const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; const ADC_FULL_SCALE_VOLTS: f64 = 3.3; const ADC_MAX_CODE: f64 = 4_095.0; -/// Longest raw history kept, in seconds of samples at the active stream rate. -const RING_SECONDS: f64 = 130.0; +/// Default monitor cache, in seconds of samples at the active stream rate +/// (user-settable 1–130 s). +const DEFAULT_CACHE_SECONDS: f64 = 20.0; +const MAX_CACHE_SECONDS: f64 = 130.0; /// Absolute sample cap guarding against absurd advertised rates (8 MiB of /// codes at most). const RING_MAX_SAMPLES: usize = 4_000_000; @@ -82,7 +87,6 @@ impl Mode { } } -#[derive(Default)] struct SharedState { /// Sample rate of the current segment (from the frame headers). rate_hz: u32, @@ -96,12 +100,31 @@ struct SharedState { resync_bytes: u64, /// Segment restarts observed (rate changes, index jumps, reconnects). segments: u64, + /// Monitor-cache length driving ring eviction (user setting). + cache_seconds: f64, error: Option, } +impl Default for SharedState { + fn default() -> Self { + Self { + rate_hz: 0, + ring_first_index: 0, + samples: VecDeque::new(), + latest: None, + device_dropped: 0, + crc_failures: 0, + resync_bytes: 0, + segments: 0, + cache_seconds: DEFAULT_CACHE_SECONDS, + error: None, + } + } +} + impl SharedState { - fn ring_capacity(rate_hz: u32) -> usize { - ((f64::from(rate_hz.max(1)) * RING_SECONDS) as usize).min(RING_MAX_SAMPLES) + fn ring_capacity(&self, rate_hz: u32) -> usize { + ((f64::from(rate_hz.max(1)) * self.cache_seconds) as usize).clamp(2, RING_MAX_SAMPLES) } /// Ingests one `SamplesU16` frame. Any discontinuity — rate change, @@ -128,7 +151,7 @@ impl SharedState { let excess = self .samples .len() - .saturating_sub(Self::ring_capacity(rate_hz)); + .saturating_sub(self.ring_capacity(rate_hz)); if excess > 0 { self.samples.drain(..excess); self.ring_first_index += excess as u64; @@ -136,6 +159,66 @@ impl SharedState { } } +/// One active disk recording: every clean `SamplesU16` frame is appended +/// verbatim to a `.pdq` file; `stop` writes the JSON sidecar next to it. +struct RecordingSink { + writer: PdqWriter, + pdq_path: PathBuf, + started_slug: String, + samples_written: u64, + write_error: Option, + /// Integrity counters at recording start, so the sidecar reports deltas + /// for exactly the recorded span. + start_crc_failures: u64, + start_resync_bytes: u64, + start_device_dropped: u32, + start_segments: u64, +} + +type SharedRecording = Arc>>; + +fn record_frame(recording: &SharedRecording, frame: &stage_a_io::Frame, samples: usize) { + let Ok(mut slot) = recording.lock() else { + return; + }; + let Some(sink) = slot.as_mut() else { + return; + }; + if sink.write_error.is_some() { + return; + } + match sink.writer.write_frame(frame) { + Ok(()) => sink.samples_written += samples as u64, + Err(err) => sink.write_error = Some(format!("recording write failed: {err}")), + } +} + +/// `YYYYmmdd_HHMMSS` in UTC without a date-time dependency (Howard Hinnant's +/// civil-from-days algorithm). +fn timestamp_slug() -> String { + let seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = (seconds / 86_400) as i64; + let (secs_of_day, z) = ((seconds % 86_400) as u32, days + 719_468); + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097) as u64; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = if month <= 2 { year + 1 } else { year }; + format!( + "{year:04}{month:02}{day:02}_{:02}{:02}{:02}", + secs_of_day / 3_600, + (secs_of_day / 60) % 60, + secs_of_day % 60 + ) +} + /// Background reader owning the stream port (or the mock generator). struct Reader { stop: Arc, @@ -147,6 +230,7 @@ impl Reader { path: String, shared: Arc>, generation: Arc, + recording: SharedRecording, ) -> Result { let port = serialport::new(&path, 115_200) .timeout(Duration::from_millis(50)) @@ -156,7 +240,7 @@ impl Reader { let thread_stop = Arc::clone(&stop); let join = std::thread::Builder::new() .name("stage-a-photodiode".into()) - .spawn(move || read_frames(port, &shared, &generation, &thread_stop)) + .spawn(move || read_frames(port, &shared, &generation, &recording, &thread_stop)) .expect("spawning the photodiode reader thread must succeed"); Ok(Self { stop, @@ -166,7 +250,11 @@ impl Reader { /// Hardware-free source: synthesizes a noisy 5 Hz sine around 1 V in /// firmware-sized blocks at the firmware's default stream rate. - fn spawn_mock(shared: Arc>, generation: Arc) -> Self { + fn spawn_mock( + shared: Arc>, + generation: Arc, + recording: SharedRecording, + ) -> Self { let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); let join = std::thread::Builder::new() @@ -174,6 +262,7 @@ impl Reader { .spawn(move || { let start = Instant::now(); let mut next_index: u64 = 0; + let mut sequence: u32 = 0; while !thread_stop.load(Ordering::Relaxed) { let target = (start.elapsed().as_secs_f64() * f64::from(MOCK_RATE_HZ)) as u64; let mut produced = false; @@ -181,6 +270,14 @@ impl Reader { let codes: Vec = (0..MOCK_BLOCK_SAMPLES) .map(|i| mock_code(next_index + i as u64)) .collect(); + // Recordings capture real wire frames; synthesize the + // identical framing so mock recordings parse the same. + record_frame( + &recording, + &mock_sample_frame(sequence, next_index, &codes), + codes.len(), + ); + sequence = sequence.wrapping_add(1); if let Ok(mut state) = shared.lock() { state.ingest(next_index, MOCK_RATE_HZ, 0, &codes); } @@ -201,6 +298,24 @@ impl Reader { } } +fn mock_sample_frame(sequence: u32, first_index: u64, codes: &[u16]) -> stage_a_io::Frame { + let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); + stage_a_io::Frame::build( + stage_a_io::FrameHeader { + version: stage_a_io::wire::PROTOCOL_VERSION, + frame_type: stage_a_io::FrameType::SamplesU16, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: first_index, + sample_rate_hz: MOCK_RATE_HZ, + dropped_samples: 0, + crc32: 0, + }, + payload, + ) +} + /// Deterministic mock sample: 1 V ± 0.5 V sine at 5 Hz plus ~20 mV of hash /// noise, so the moving-average indicator has something to smooth. fn mock_code(index: u64) -> u16 { @@ -225,6 +340,7 @@ fn read_frames( mut port: Box, shared: &Mutex, generation: &AtomicU64, + recording: &SharedRecording, stop: &AtomicBool, ) { let mut parser = FrameParser::default(); @@ -251,6 +367,7 @@ fn read_frames( let Some(codes) = frame.samples() else { continue; // Control/summary frames are not expected here. }; + record_frame(recording, &frame, codes.len()); if let Ok(mut state) = shared.lock() { state.ingest( frame.header.first_sample_index, @@ -284,7 +401,10 @@ pub struct StageAPhotodiodePlugin { reader: Option, shared: Arc>, generation: Arc, + recording: SharedRecording, last_error: Option, + /// One-line feedback about the most recent save/recording action. + last_save_note: Option, // -- settings -- connect_requested: bool, port_hint: String, @@ -293,6 +413,7 @@ pub struct StageAPhotodiodePlugin { window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, + data_dir: String, } impl Default for StageAPhotodiodePlugin { @@ -302,7 +423,9 @@ impl Default for StageAPhotodiodePlugin { reader: None, shared: Arc::new(Mutex::new(SharedState::default())), generation: Arc::new(AtomicU64::new(1)), + recording: Arc::new(Mutex::new(None)), last_error: None, + last_save_note: None, connect_requested: false, port_hint: "auto".into(), mode: Mode::Raw, @@ -310,6 +433,7 @@ impl Default for StageAPhotodiodePlugin { window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, + data_dir: String::new(), } } } @@ -331,6 +455,7 @@ impl StageAPhotodiodePlugin { self.reader = Some(Reader::spawn_mock( Arc::clone(&self.shared), Arc::clone(&self.generation), + Arc::clone(&self.recording), )); return; } @@ -345,7 +470,12 @@ impl StageAPhotodiodePlugin { } else { self.port_hint.clone() }; - match Reader::spawn_serial(path, Arc::clone(&self.shared), Arc::clone(&self.generation)) { + match Reader::spawn_serial( + path, + Arc::clone(&self.shared), + Arc::clone(&self.generation), + Arc::clone(&self.recording), + ) { Ok(reader) => self.reader = Some(reader), Err(err) => { self.last_error = Some(err); @@ -360,6 +490,183 @@ impl StageAPhotodiodePlugin { self.generation.fetch_add(1, Ordering::Relaxed); } + fn recording_active(&self) -> bool { + self.recording + .lock() + .map(|slot| slot.is_some()) + .unwrap_or(false) + } + + fn resolved_data_dir(&self) -> Result { + if self.data_dir.trim().is_empty() { + return Err("set the data directory first (Data section)".into()); + } + Ok(PathBuf::from(self.data_dir.trim())) + } + + fn start_recording(&mut self) -> Result<(), String> { + if self.recording_active() { + return Ok(()); + } + let dir = self.resolved_data_dir()?; + let slug = timestamp_slug(); + let pdq_path = dir.join(format!("pd_rec_{slug}.pdq")); + let writer = PdqWriter::create(&pdq_path) + .map_err(|err| format!("creating {} failed: {err}", pdq_path.display()))?; + let (crc, resync, dropped, segments) = match self.shared.lock() { + Ok(state) => ( + state.crc_failures, + state.resync_bytes, + state.device_dropped, + state.segments, + ), + Err(_) => (0, 0, 0, 0), + }; + let sink = RecordingSink { + writer, + pdq_path: pdq_path.clone(), + started_slug: slug, + samples_written: 0, + write_error: None, + start_crc_failures: crc, + start_resync_bytes: resync, + start_device_dropped: dropped, + start_segments: segments, + }; + if let Ok(mut slot) = self.recording.lock() { + *slot = Some(sink); + } + self.last_save_note = Some(format!("recording → {}", pdq_path.display())); + Ok(()) + } + + fn stop_recording(&mut self) -> Result<(), String> { + let Some(sink) = self.recording.lock().ok().and_then(|mut slot| slot.take()) else { + return Ok(()); + }; + let (rate_hz, crc, resync, dropped, segments) = match self.shared.lock() { + Ok(state) => ( + state.rate_hz, + state.crc_failures, + state.resync_bytes, + state.device_dropped, + state.segments, + ), + Err(_) => (0, 0, 0, 0, 0), + }; + let integrity = StreamIntegrity { + skipped_bytes: resync.saturating_sub(sink.start_resync_bytes), + crc_failures: crc.saturating_sub(sink.start_crc_failures), + sequence_gaps: segments.saturating_sub(sink.start_segments), + dropped_samples: u64::from(dropped.saturating_sub(sink.start_device_dropped)), + }; + let write_error = sink.write_error.clone(); + let started = sink.started_slug.clone(); + let samples = sink.samples_written; + let summary = sink + .writer + .finish(integrity) + .map_err(|err| format!("finishing recording failed: {err}"))?; + let sidecar = json!({ + "kind": "recording", + "started_utc": started, + "stopped_utc": timestamp_slug(), + "port": self.port_hint, + "sample_rate_hz": rate_hz, + "samples_written": samples, + "pdq_path": summary.path, + "pdq_frames": summary.frames_written, + "pdq_bytes": summary.bytes_written, + "pdq_crc32": summary.file_crc32, + "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, + "display_mode": self.mode.name(), + "reference_volts": self.reference_volts, + "integrity": { + "resync_bytes": summary.integrity.skipped_bytes, + "crc_failures": summary.integrity.crc_failures, + "segment_restarts": summary.integrity.sequence_gaps, + "device_dropped_samples": summary.integrity.dropped_samples, + }, + "valid": summary.valid && write_error.is_none(), + "write_error": write_error, + }); + let sidecar_path = sink.pdq_path.with_extension("json"); + write_json(&sidecar_path, &sidecar)?; + self.last_save_note = Some(format!( + "saved recording {} ({} samples)", + sink.pdq_path.display(), + samples + )); + Ok(()) + } + + /// Dumps the current monitor cache (ring) as CSV + JSON sidecar. Raw + /// codes and raw volts only — mode/reference land in the sidecar so + /// EXCITATION values stay derivable without baking display state into + /// the data. + fn save_cache_snapshot(&mut self) -> Result<(), String> { + let dir = self.resolved_data_dir()?; + let slug = timestamp_slug(); + let csv_path = dir.join(format!("pd_cache_{slug}.csv")); + let state = self + .shared + .lock() + .map_err(|_| "photodiode state lock poisoned".to_owned())?; + if state.samples.is_empty() || state.rate_hz == 0 { + return Err("no samples cached yet".into()); + } + std::fs::create_dir_all(&dir) + .map_err(|err| format!("creating {} failed: {err}", dir.display()))?; + let file = File::create(&csv_path) + .map_err(|err| format!("creating {} failed: {err}", csv_path.display()))?; + let mut writer = BufWriter::new(file); + let rate = f64::from(state.rate_hz); + writeln!(writer, "sample_index,t_s,code,volts") + .map_err(|err| format!("writing CSV failed: {err}"))?; + for (offset, &code) in state.samples.iter().enumerate() { + let index = state.ring_first_index + offset as u64; + writeln!( + writer, + "{index},{:.9},{code},{:.6}", + index as f64 / rate, + code_to_volts(f64::from(code)) + ) + .map_err(|err| format!("writing CSV failed: {err}"))?; + } + writer + .flush() + .map_err(|err| format!("writing CSV failed: {err}"))?; + + let sidecar = json!({ + "kind": "cache_snapshot", + "created_utc": slug, + "port": self.port_hint, + "sample_rate_hz": state.rate_hz, + "samples": state.samples.len(), + "first_sample_index": state.ring_first_index, + "cache_seconds": state.cache_seconds, + "csv_path": csv_path, + "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, + "display_mode": self.mode.name(), + "reference_volts": self.reference_volts, + "time_base": "t_s = sample_index / sample_rate_hz, device clock, segment-relative", + "integrity": { + "resync_bytes": state.resync_bytes, + "crc_failures": state.crc_failures, + "segment_restarts": state.segments, + "device_dropped_samples": state.device_dropped, + }, + }); + let sample_count = state.samples.len(); + drop(state); + write_json(&csv_path.with_extension("json"), &sidecar)?; + self.last_save_note = Some(format!( + "saved cache {} ({sample_count} samples)", + csv_path.display() + )); + Ok(()) + } + /// Value shown for one sample under the current mode, in volts. fn display_volts(&self, code: f64) -> f64 { match self.mode { @@ -616,6 +923,12 @@ impl StageAPhotodiodePlugin { } } +fn write_json(path: &Path, value: &Value) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| format!("serializing sidecar failed: {err}"))?; + std::fs::write(path, bytes).map_err(|err| format!("writing {} failed: {err}", path.display())) +} + fn serial_ports() -> Vec { serialport::available_ports() .map(|ports| { @@ -747,6 +1060,11 @@ impl Plugin for StageAPhotodiodePlugin { self.enabled = enabled; if !enabled { self.connect_requested = false; + // Finalize an active recording so the .pdq/.json pair is complete + // even when the plugin is disabled mid-run. + if let Err(err) = self.stop_recording() { + self.last_error = Some(err); + } self.disconnect(); } } @@ -783,114 +1101,184 @@ impl Plugin for StageAPhotodiodePlugin { .position(|m| *m == self.mode) .unwrap_or(0); SettingsSchema { - sections: vec![SettingsSection { - label: "Photodiode readout".into(), - description: Some( - "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ + sections: vec![ + SettingsSection { + label: "Photodiode readout".into(), + description: Some( + "Reads the free-running PDA1 sample stream on the Teensy's SECOND serial \ port (firmware 0.4.0+, 20 kSa/s default). EXCITATION shows \ I_exc = I_tot − I_pd: the diode sits behind the PBS and sees the light \ removed from the excitation beam." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "auto (recommended) listens on the attached usbmodem ports and \ + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "auto (recommended) listens on the attached usbmodem ports and \ picks the one streaming PDA1 sample frames — the Teensy stream \ port; mock = synthetic data" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, }, - }, - SettingItem { - key: "connect".into(), - label: "Connect".into(), - tooltip: Some( - "Opens/closes the stream port (read-only, no camera required).".into(), - ), - kind: SettingKind::Bool { - default: self.connect_requested, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the stream port (read-only, no camera required)." + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, }, - }, - SettingItem { - key: "mode".into(), - label: "Mode".into(), - tooltip: Some( - "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd".into(), - ), - kind: SettingKind::Enum { - variants: mode_variants, - default: mode_default, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some( + "RAW: ADC code and volts as measured. EXCITATION: I_tot − I_pd" + .into(), + ), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, }, - }, - SettingItem { - key: "reference_volts".into(), - label: "Reference I_tot".into(), - tooltip: Some( - "Total power reference for EXCITATION mode, in photodiode volts: \ + SettingItem { + key: "reference_volts".into(), + label: "Reference I_tot".into(), + tooltip: Some( + "Total power reference for EXCITATION mode, in photodiode volts: \ the PD reading with the full beam diverted into the diode" - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: ADC_FULL_SCALE_VOLTS, - speed: 0.01, - default: self.reference_volts, + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: ADC_FULL_SCALE_VOLTS, + speed: 0.01, + default: self.reference_volts, + }, }, - }, - SettingItem { - key: "window_s".into(), - label: "Chart window".into(), - tooltip: Some( - "Seconds of history shown in the live chart. Short windows \ + SettingItem { + key: "window_s".into(), + label: "Chart window".into(), + tooltip: Some( + "Seconds of history shown in the live chart. Short windows \ (≤ 50 ms) resolve individual modulation cycles at 20 kSa/s." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.01, - max: 120.0, - speed: 0.05, - default: self.window_s, + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 120.0, + speed: 0.05, + default: self.window_s, + }, }, - }, - SettingItem { - key: "avg_samples".into(), - label: "Average window".into(), - tooltip: Some( - "Moving-average window in samples (1 = off). Ignored while \ + SettingItem { + key: "avg_samples".into(), + label: "Average window".into(), + tooltip: Some( + "Moving-average window in samples (1 = off). Ignored while \ 'Average sync frequency' is set." - .into(), - ), - kind: SettingKind::I64Drag { - min: 1, - max: 1_000_000, - default: self.avg_samples as i64, + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 1_000_000, + default: self.avg_samples as i64, + }, }, - }, - SettingItem { - key: "avg_sync_freq_hz".into(), - label: "Average sync frequency".into(), - tooltip: Some( - "0 = off. When set to the modulation frequency (Hz), the moving \ + SettingItem { + key: "avg_sync_freq_hz".into(), + label: "Average sync frequency".into(), + tooltip: Some( + "0 = off. When set to the modulation frequency (Hz), the moving \ average spans exactly one full period (window = rate / f), so the \ mean level no longer depends on the modulation phase." - .into(), - ), - kind: SettingKind::F64Drag { - min: 0.0, - max: 100_000.0, - speed: 1.0, - default: self.avg_sync_freq_hz, + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 100_000.0, + speed: 1.0, + default: self.avg_sync_freq_hz, + }, }, - }, - ], - }], + ], + }, + SettingsSection { + label: "Data".into(), + description: Some( + "Monitor cache and disk recording. The cache always holds the last \ + N seconds; recording tees every incoming frame to a .pdq file \ + (+ JSON sidecar) so length is disk-bound. CSV/PDQ store raw codes \ + and raw volts on the device clock; mode and reference go into the \ + sidecar." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "data_dir".into(), + label: "Data directory".into(), + tooltip: Some( + "Where recordings and cache snapshots are written.".into(), + ), + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.data_dir.clone(), + }, + }, + SettingItem { + key: "cache_s".into(), + label: "Cache length".into(), + tooltip: Some( + "Seconds of raw samples kept in memory for the chart and \ + cache snapshots." + .into(), + ), + kind: SettingKind::F64Drag { + min: 1.0, + max: MAX_CACHE_SECONDS, + speed: 1.0, + default: self + .shared + .lock() + .map(|state| state.cache_seconds) + .unwrap_or(DEFAULT_CACHE_SECONDS), + }, + }, + SettingItem { + key: "record".into(), + label: "Record to disk".into(), + tooltip: Some( + "Start/stop appending every incoming sample frame to \ + pd_rec_.pdq; stopping writes the JSON sidecar." + .into(), + ), + kind: SettingKind::Bool { + default: self.recording_active(), + }, + }, + SettingItem { + key: "save_snapshot".into(), + label: "Save cache snapshot".into(), + tooltip: Some( + "Write the current cache as pd_cache_.csv \ + (+ JSON sidecar)." + .into(), + ), + kind: SettingKind::Button, + }, + ], + }, + ], } } @@ -917,6 +1305,15 @@ impl Plugin for StageAPhotodiodePlugin { "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), + "data_dir" => Some(json!(self.data_dir)), + "cache_s" => Some(json!(self + .shared + .lock() + .map(|state| state.cache_seconds) + .unwrap_or(DEFAULT_CACHE_SECONDS))), + "record" => Some(json!(self.recording_active())), + // Momentary trigger: never reports as pressed. + "save_snapshot" => Some(json!(false)), _ => None, } } @@ -965,6 +1362,45 @@ impl Plugin for StageAPhotodiodePlugin { self.avg_sync_freq_hz = freq.clamp(0.0, 100_000.0); Ok(()) } + "data_dir" => { + self.data_dir = value + .as_str() + .ok_or("data_dir must be a string")? + .to_owned(); + Ok(()) + } + "cache_s" => { + let seconds = value.as_f64().ok_or("cache_s must be a number")?; + if let Ok(mut state) = self.shared.lock() { + state.cache_seconds = seconds.clamp(1.0, MAX_CACHE_SECONDS); + } + Ok(()) + } + "record" => { + let requested = value.as_bool().ok_or("record must be a boolean")?; + // Failures surface through status entries (like `connect`), + // so a missing data directory doesn't read as a broken UI. + let result = if requested { + self.start_recording() + } else { + self.stop_recording() + }; + if let Err(err) = result { + self.last_error = Some(err); + } else { + self.last_error = None; + } + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + "save_snapshot" => { + match self.save_cache_snapshot() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(()) + } _ => Err(format!("unknown setting: {key}")), } } @@ -1012,6 +1448,25 @@ impl Plugin for StageAPhotodiodePlugin { ))); } } + if self.recording_active() { + let (samples, path) = self + .recording + .lock() + .ok() + .and_then(|slot| { + slot.as_ref() + .map(|sink| (sink.samples_written, sink.pdq_path.display().to_string())) + }) + .unwrap_or((0, String::new())); + let seconds = if rate_hz > 0 { + samples as f64 / f64::from(rate_hz) + } else { + 0.0 + }; + entries.push(StatusEntry::Text(format!("● REC {seconds:.1} s → {path}"))); + } else if let Some(note) = &self.last_save_note { + entries.push(StatusEntry::Text(note.clone())); + } if let Some(error) = stream_error.or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } @@ -1146,8 +1601,9 @@ mod tests { #[test] fn ring_is_bounded_by_duration() { let mut state = SharedState::default(); - let rate = 1_000; // capacity = 130_000 samples - let cap = SharedState::ring_capacity(rate); + let rate = 1_000; // capacity = cache_seconds (20 s default) × rate + let cap = state.ring_capacity(rate); + assert_eq!(cap, 20_000, "default cache is 20 s"); let block: Vec = (0..1_000).map(|i| (i % 4_096) as u16).collect(); let mut index = 0_u64; for _ in 0..(cap / block.len() + 5) { @@ -1276,6 +1732,122 @@ mod tests { plugin.disconnect(); } + fn temp_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "stage-a-photodiode-{tag}-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + #[test] + fn cache_snapshot_writes_csv_and_sidecar() { + let dir = temp_dir("snapshot"); + let mut plugin = StageAPhotodiodePlugin::default(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(10, 20_000, 0, &[100, 200, 300]); + } + plugin.set_setting("save_snapshot", json!(true)).unwrap(); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + + let mut csv_files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|ext| ext == "csv")) + .collect(); + assert_eq!(csv_files.len(), 1); + let csv_path = csv_files.pop().unwrap(); + let csv = std::fs::read_to_string(&csv_path).unwrap(); + let mut lines = csv.lines(); + assert_eq!(lines.next(), Some("sample_index,t_s,code,volts")); + let first = lines.next().unwrap(); + assert!(first.starts_with("10,0.000500000,100,"), "{first}"); + assert_eq!(csv.lines().count(), 4, "header + 3 samples"); + + let sidecar: Value = + serde_json::from_slice(&std::fs::read(csv_path.with_extension("json")).unwrap()) + .unwrap(); + assert_eq!(sidecar["kind"], "cache_snapshot"); + assert_eq!(sidecar["sample_rate_hz"], 20_000); + assert_eq!(sidecar["samples"], 3); + + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn snapshot_without_data_dir_reports_an_error() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("save_snapshot", json!(true)).unwrap(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("data directory"))); + } + + #[test] + fn recording_tees_frames_to_pdq_and_writes_a_sidecar() { + let dir = temp_dir("recording"); + let mut plugin = StageAPhotodiodePlugin::default(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + plugin.set_setting("record", json!(true)).unwrap(); + assert!(plugin.recording_active()); + assert_eq!(plugin.get_setting("record"), Some(json!(true))); + + // The reader thread path: every parsed frame is teed to the sink. + let frame = mock_sample_frame(0, 0, &[1, 2, 3, 4]); + record_frame(&plugin.recording, &frame, 4); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(0, MOCK_RATE_HZ, 0, &[1, 2, 3, 4]); + } + + plugin.set_setting("record", json!(false)).unwrap(); + assert!(!plugin.recording_active()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + + let pdq_path: std::path::PathBuf = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .find(|p| p.extension().is_some_and(|ext| ext == "pdq")) + .expect("pdq written"); + assert_eq!(std::fs::read(&pdq_path).unwrap(), frame.to_bytes()); + + let sidecar: Value = + serde_json::from_slice(&std::fs::read(pdq_path.with_extension("json")).unwrap()) + .unwrap(); + assert_eq!(sidecar["kind"], "recording"); + assert_eq!(sidecar["samples_written"], 4); + assert_eq!(sidecar["pdq_frames"], 1); + assert_eq!(sidecar["valid"], true); + + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn cache_length_setting_drives_ring_capacity() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("cache_s", json!(2.0)).unwrap(); + assert_eq!(plugin.get_setting("cache_s"), Some(json!(2.0))); + let mut state = plugin.shared.lock().unwrap(); + assert_eq!(state.ring_capacity(1_000), 2_000); + let block: Vec = vec![1; 1_000]; + for i in 0..5_u64 { + let first = i * 1_000; + state.ingest(first, 1_000, 0, &block); + } + assert_eq!(state.samples.len(), 2_000); + } + /// The host settings UI exchanges enum values as indices into the /// schema's variant list (radio buttons send `json!(index)`). #[test] From 0cd8be933715ec536ba4f44ab90896ae802a825e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 16:59:45 +0200 Subject: [PATCH 17/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20spectr?= =?UTF-8?q?um=20view=20and=20absolute-time=20axis=20to=20the=20photodiode?= =?UTF-8?q?=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PD Spectrum window: Hann-windowed radix-2 FFT (no deps) over the newest power-of-two window of raw samples (256-16384; ≈1.2 Hz resolution at 20 kSa/s), one-sided amplitude in volts with max-hold bin decimation so narrow peaks survive the plot budget. Window placement means the FFT only runs while the window is open. Verified by test: a synthesized 1 kHz 0.4 V tone is recovered at the right frequency and amplitude. - time_axis setting: BEFORE NOW (scrolling, x ends at 0) or SEGMENT TIME (absolute device-clock seconds) — frozen plots and cursor measurements read as positions instead of implied motion. --- plugins/stage-a-photodiode/src/lib.rs | 291 +++++++++++++++++++++++++- 1 file changed, 286 insertions(+), 5 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 6f91025..ffc2d48 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -42,6 +42,8 @@ use serde_json::{json, Value}; use stage_a_io::{FrameParser, ParseEvent, PdqWriter, StreamIntegrity}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; +const SPECTRUM_DATASET_ID: &str = "stage-a-photodiode.spectrum"; +const SPECTRUM_VIEW_ID: &str = "stage-a-photodiode.spectrum.view"; const SERIES_VIEW_ID: &str = "stage-a-photodiode.series.view"; const STATUS_DATASET_ID: &str = "stage-a-photodiode.status"; const STATUS_VIEW_ID: &str = "stage-a-photodiode.status.view"; @@ -58,6 +60,10 @@ const RING_MAX_SAMPLES: usize = 4_000_000; /// Envelope buckets per rendered chart line; keeps the plot payload bounded /// no matter how many raw samples the window covers. const MAX_PLOT_BUCKETS: usize = 1_000; +/// Spectrum FFT window bounds: 16384 samples ≈ 0.8 s at 20 kSa/s +/// (Δf ≈ 1.2 Hz); below 256 samples a spectrum is not meaningful. +const SPECTRUM_MIN_SAMPLES: usize = 256; +const SPECTRUM_MAX_SAMPLES: usize = 16_384; /// The firmware's default stream rate; the mock mirrors it. const MOCK_RATE_HZ: u32 = 20_000; const MOCK_BLOCK_SAMPLES: usize = 256; @@ -87,6 +93,37 @@ impl Mode { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TimeAxis { + /// Scrolling view: x = seconds before the newest sample (ends at 0). + BeforeNow, + /// Fixed view: x = seconds since the segment start on the device clock — + /// a frozen plot reads as absolute positions, not implied motion. + Segment, +} + +impl TimeAxis { + const VARIANTS: [TimeAxis; 2] = [TimeAxis::BeforeNow, TimeAxis::Segment]; + + fn name(self) -> &'static str { + match self { + Self::BeforeNow => "BEFORE NOW", + Self::Segment => "SEGMENT TIME", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|axis| axis.name() == name) + } + + fn label(self) -> &'static str { + match self { + Self::BeforeNow => "time before now [s]", + Self::Segment => "segment time [s]", + } + } +} + struct SharedState { /// Sample rate of the current segment (from the frame headers). rate_hz: u32, @@ -413,6 +450,7 @@ pub struct StageAPhotodiodePlugin { window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, + time_axis: TimeAxis, data_dir: String, } @@ -433,6 +471,7 @@ impl Default for StageAPhotodiodePlugin { window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, + time_axis: TimeAxis::BeforeNow, data_dir: String::new(), } } @@ -710,8 +749,9 @@ impl StageAPhotodiodePlugin { Mode::Raw => "photodiode", Mode::Excitation => "excitation", }; + let x_label = self.time_axis.label(); let empty = |label: &str| Series1dV1 { - x_label: "time before now [s]".into(), + x_label: x_label.into(), y_label: label.into(), lines: vec![Series1dLine { name: trace_name.into(), @@ -772,8 +812,11 @@ impl StageAPhotodiodePlugin { } } if bucket_n == bucket_len || i == total - 1 { - let x = (state.ring_first_index + i as u64) as f64 / rate - - latest_x_index as f64 / rate; + let device_t = (state.ring_first_index + i as u64) as f64 / rate; + let x = match self.time_axis { + TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, + TimeAxis::Segment => device_t, + }; mean_points.push(Series1dPoint { x, y: self.display_volts(bucket_sum as f64 / bucket_n as f64), @@ -827,12 +870,92 @@ impl StageAPhotodiodePlugin { }); } Series1dV1 { - x_label: "time before now [s]".into(), + x_label: x_label.into(), y_label: y_label.into(), lines, } } + /// Amplitude spectrum of the newest power-of-two window of raw samples + /// (Hann-windowed radix-2 FFT). Only computed while the spectrum window + /// is open — it has Window placement, and the host fetches datasets of + /// closed windows never. + fn spectrum_dataset(&self) -> Series1dV1 { + let empty = Series1dV1 { + x_label: "frequency [Hz]".into(), + y_label: "amplitude [V]".into(), + lines: vec![Series1dLine { + name: "spectrum".into(), + points: Vec::new(), + }], + }; + let Ok(state) = self.shared.lock() else { + return empty; + }; + let total = state.samples.len(); + if total < SPECTRUM_MIN_SAMPLES || state.rate_hz == 0 { + return empty; + } + let available = total.min(SPECTRUM_MAX_SAMPLES); + let n = if available.is_power_of_two() { + available + } else { + available.next_power_of_two() >> 1 + }; + let start = total - n; + let mut real: Vec = state + .samples + .range(start..) + .map(|&code| code_to_volts(f64::from(code))) + .collect(); + let rate = f64::from(state.rate_hz); + drop(state); + + let mean = real.iter().sum::() / n as f64; + // Hann window (coherent gain 0.5) on the demeaned signal. + for (i, value) in real.iter_mut().enumerate() { + let w = 0.5 * (1.0 - (2.0 * std::f64::consts::PI * i as f64 / (n as f64 - 1.0)).cos()); + *value = (*value - mean) * w; + } + let mut imag = vec![0.0_f64; n]; + fft_radix2(&mut real, &mut imag); + + // One-sided amplitude: 2·|X|/(N·0.5); decimate bins by max-hold so + // narrow peaks survive the plot budget. + let bins = n / 2; + let bucket = bins.div_ceil(MAX_PLOT_BUCKETS).max(1); + let mut points = Vec::with_capacity(bins.div_ceil(bucket)); + let mut peak = 0.0_f64; + let mut peak_freq = 0.0_f64; + let mut in_bucket = 0_usize; + for k in 1..bins { + let amplitude = 2.0 * (real[k] * real[k] + imag[k] * imag[k]).sqrt() / (n as f64 * 0.5); + let freq = k as f64 * rate / n as f64; + if amplitude > peak { + peak = amplitude; + peak_freq = freq; + } + in_bucket += 1; + if in_bucket == bucket || k == bins - 1 { + points.push(Series1dPoint { + x: peak_freq, + y: peak, + }); + peak = 0.0; + peak_freq = freq; + in_bucket = 0; + } + } + Series1dV1 { + x_label: "frequency [Hz]".into(), + y_label: "amplitude [V]".into(), + lines: vec![Series1dLine { + name: format!("spectrum ({n} spl, Δf {:.2} Hz)", rate / n as f64), + points, + }], + } + } + fn status_dataset(&self) -> TableDatasetV1 { let (latest, rate_hz, average, integrity, stream_error) = match self.shared.lock() { Ok(state) => ( @@ -923,6 +1046,51 @@ impl StageAPhotodiodePlugin { } } +/// In-place iterative radix-2 Cooley–Tukey FFT. Lengths must be powers of +/// two; sized for the spectrum window (≤ 16384), where it runs in well under +/// a millisecond. +fn fft_radix2(real: &mut [f64], imag: &mut [f64]) { + let n = real.len(); + debug_assert!(n.is_power_of_two() && imag.len() == n); + // Bit-reversal permutation. + let mut j = 0_usize; + for i in 1..n { + let mut bit = n >> 1; + while j & bit != 0 { + j ^= bit; + bit >>= 1; + } + j |= bit; + if i < j { + real.swap(i, j); + imag.swap(i, j); + } + } + let mut len = 2_usize; + while len <= n { + let angle = -2.0 * std::f64::consts::PI / len as f64; + let (step_r, step_i) = (angle.cos(), angle.sin()); + for start in (0..n).step_by(len) { + let (mut w_r, mut w_i) = (1.0_f64, 0.0_f64); + for k in start..start + len / 2 { + let (even_r, even_i) = (real[k], imag[k]); + let (odd_r, odd_i) = ( + real[k + len / 2] * w_r - imag[k + len / 2] * w_i, + real[k + len / 2] * w_i + imag[k + len / 2] * w_r, + ); + real[k] = even_r + odd_r; + imag[k] = even_i + odd_i; + real[k + len / 2] = even_r - odd_r; + imag[k + len / 2] = even_i - odd_i; + let next_r = w_r * step_r - w_i * step_i; + w_i = w_r * step_i + w_i * step_r; + w_r = next_r; + } + } + len <<= 1; + } +} + fn write_json(path: &Path, value: &Value) -> Result<(), String> { let bytes = serde_json::to_vec_pretty(value) .map_err(|err| format!("serializing sidecar failed: {err}"))?; @@ -1210,6 +1378,26 @@ impl Plugin for StageAPhotodiodePlugin { default: self.avg_sync_freq_hz, }, }, + SettingItem { + key: "time_axis".into(), + label: "Time axis".into(), + tooltip: Some( + "BEFORE NOW scrolls (x ends at 0); SEGMENT TIME shows absolute \ + seconds on the device clock — better for frozen plots and \ + cursor measurements." + .into(), + ), + kind: SettingKind::Enum { + variants: TimeAxis::VARIANTS + .iter() + .map(|axis| axis.name().to_owned()) + .collect(), + default: TimeAxis::VARIANTS + .iter() + .position(|axis| *axis == self.time_axis) + .unwrap_or(0), + }, + }, ], }, SettingsSection { @@ -1305,6 +1493,13 @@ impl Plugin for StageAPhotodiodePlugin { "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), + "time_axis" => { + let index = TimeAxis::VARIANTS + .iter() + .position(|axis| *axis == self.time_axis) + .unwrap_or(0); + Some(json!(index)) + } "data_dir" => Some(json!(self.data_dir)), "cache_s" => Some(json!(self .shared @@ -1362,6 +1557,16 @@ impl Plugin for StageAPhotodiodePlugin { self.avg_sync_freq_hz = freq.clamp(0.0, 100_000.0); Ok(()) } + "time_axis" => { + let names: Vec = TimeAxis::VARIANTS + .iter() + .map(|axis| axis.name().to_owned()) + .collect(); + let name = enum_choice(&value, &names)?; + self.time_axis = TimeAxis::from_name(&name) + .ok_or_else(|| format!("unknown time axis: {name}"))?; + Ok(()) + } "data_dir" => { self.data_dir = value .as_str() @@ -1484,6 +1689,16 @@ impl Plugin for StageAPhotodiodePlugin { display: None, relations: Vec::new(), }, + HostDatasetDescriptor { + id: SPECTRUM_DATASET_ID.into(), + title: "Photodiode spectrum".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Not enough samples for a spectrum yet — connect the stream \ + port and wait a moment." + .into(), + display: None, + relations: Vec::new(), + }, HostDatasetDescriptor { id: STATUS_DATASET_ID.into(), title: "Photodiode readout".into(), @@ -1501,6 +1716,13 @@ impl Plugin for StageAPhotodiodePlugin { placement: HostViewPlacement::Window, kind: HostViewKind::LineSeriesWindow, }, + HostViewDescriptor { + id: SPECTRUM_VIEW_ID.into(), + title: "PD Spectrum".into(), + dataset_id: SPECTRUM_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, HostViewDescriptor { id: STATUS_VIEW_ID.into(), title: "Photodiode readout".into(), @@ -1516,6 +1738,7 @@ impl Plugin for StageAPhotodiodePlugin { fn host_view_dataset(&self, dataset_id: &str) -> Option> { match dataset_id { SERIES_DATASET_ID => serde_json::to_vec(&self.series_dataset()).ok(), + SPECTRUM_DATASET_ID => serde_json::to_vec(&self.spectrum_dataset()).ok(), STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), _ => None, } @@ -1523,7 +1746,9 @@ impl Plugin for StageAPhotodiodePlugin { fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { match dataset_id { - SERIES_DATASET_ID | STATUS_DATASET_ID => self.generation.load(Ordering::Relaxed).max(1), + SERIES_DATASET_ID | SPECTRUM_DATASET_ID | STATUS_DATASET_ID => { + self.generation.load(Ordering::Relaxed).max(1) + } _ => 0, } } @@ -1732,6 +1957,62 @@ mod tests { plugin.disconnect(); } + #[test] + fn spectrum_finds_a_synthesized_tone() { + let plugin = StageAPhotodiodePlugin::default(); + let rate = 20_000_u32; + // 1 kHz, 0.4 V amplitude around 1 V — well inside the ADC range. + let codes: Vec = (0..16_384_u64) + .map(|i| { + let t = i as f64 / f64::from(rate); + let volts = 1.0 + 0.4 * (2.0 * std::f64::consts::PI * 1_000.0 * t).sin(); + (volts * ADC_MAX_CODE / ADC_FULL_SCALE_VOLTS) as u16 + }) + .collect(); + plugin.shared.lock().unwrap().ingest(0, rate, 0, &codes); + let spectrum = plugin.spectrum_dataset(); + let points = &spectrum.lines[0].points; + assert!(!points.is_empty()); + let peak = points + .iter() + .max_by(|a, b| a.y.partial_cmp(&b.y).unwrap()) + .unwrap(); + assert!( + (peak.x - 1_000.0).abs() < 5.0, + "peak at {} Hz, expected 1 kHz", + peak.x + ); + assert!( + (peak.y - 0.4).abs() < 0.05, + "peak amplitude {} V, expected ≈0.4 V", + peak.y + ); + } + + #[test] + fn segment_time_axis_uses_absolute_device_time() { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_setting("avg_samples", json!(1)).unwrap(); + plugin + .set_setting("time_axis", json!("SEGMENT TIME")) + .unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(40_000, 20_000, 0, &[1, 2, 3, 4]); + } + let series = plugin.series_dataset(); + assert_eq!(series.x_label, "segment time [s]"); + let first = series.lines[0].points.first().unwrap(); + // Sample index 40_000 at 20 kSa/s = 2 s into the segment. + assert!((first.x - 2.0).abs() < 1e-6, "got {}", first.x); + // Default mode still ends at zero. + plugin + .set_setting("time_axis", json!("BEFORE NOW")) + .unwrap(); + let series = plugin.series_dataset(); + assert!(series.lines[0].points.last().unwrap().x.abs() < 1e-9); + } + fn temp_dir(tag: &str) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!( "stage-a-photodiode-{tag}-{}", From 8d7c7eb8d5dd566d3b4c91cea6d2c6a6d982a805 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 17:04:39 +0200 Subject: [PATCH 18/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20a=20TO?= =?UTF-8?q?ML=20protocol=20executor=20to=20the=20modulation=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Timed MOD sequences without cross-plugin control: the executor lives inside the plugin that already owns the command port. - protocol file: loops = N plus [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, min, frequency_hz — fully validated before the run starts (ranges match the firmware grammar) - executor thread walks the steps on an absolute schedule (no drift accumulation) and feeds the same coalescing pending-command slot the device thread drains, so it never touches the serial port itself; the last step holds after completion (set-and-hold), stop is immediate, and disconnecting aborts the run - protocol_path uses the new Path setting kind; run/stop is a settings-driven toggle that works with no camera; progress (loop, step, summary) shows in the status entries Verified: cargo fmt, clippy -D warnings, 8 plugin tests green incl. an end-to-end run against the mock controller. --- plugins/stage-a-modulation/Cargo.toml | 1 + plugins/stage-a-modulation/src/lib.rs | 617 ++++++++++++++++++++++---- 2 files changed, 528 insertions(+), 90 deletions(-) diff --git a/plugins/stage-a-modulation/Cargo.toml b/plugins/stage-a-modulation/Cargo.toml index 2b04e93..983a927 100644 --- a/plugins/stage-a-modulation/Cargo.toml +++ b/plugins/stage-a-modulation/Cargo.toml @@ -13,3 +13,4 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true stage-a-io = { path = "../../stage-a-io" } +toml = "0.8" diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 87fc75f..4c744ce 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -28,9 +28,9 @@ use std::time::{Duration, Instant}; use augur_plugin_api::{ export_plugin, EventStoreHandle, ExecutionMode, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, - HostViewRegistry, Plugin, PluginFrame, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, - TableSchema, TableValueType, + HostViewRegistry, PathDialogKind, Plugin, PluginFrame, SettingItem, SettingKind, + SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, + TableDatasetV1, TableSchema, TableValueType, }; use serde_json::{json, Value}; use stage_a_io::{Command, MockController, StageAClient, Transport}; @@ -242,10 +242,197 @@ fn apply_reply( shared.bump(); } +/// One validated protocol step: the exact MOD command plus how long to hold +/// it before advancing. +#[derive(Debug, Clone, PartialEq)] +struct ProtocolStep { + duration: Duration, + command: Command, + summary: String, +} + +#[derive(Debug, Clone, Default)] +struct ProtocolProgress { + loops: usize, + total_steps: usize, + /// 1-based while running. + loop_index: usize, + step_index: usize, + summary: String, + finished: bool, + stopped: bool, +} + +/// Running protocol executor; dropping it stops the thread. Commands go +/// through the same coalescing pending slot the device thread drains, so the +/// executor never touches the serial port itself. +struct ProtocolRun { + stop: Arc, + join: Option>, + progress: Arc>, +} + +impl Drop for ProtocolRun { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + +/// Parses the TOML protocol format: +/// +/// ```toml +/// loops = 2 # optional, default 1 +/// [[steps]] +/// duration_s = 5.0 +/// wave = "SINE" # OFF | CONST | SINE | SQUARE +/// level = 2000 # required unless OFF +/// min = 0 # optional, periodic only +/// frequency_hz = 100.0 # required for SINE/SQUARE (0.01–2000) +/// ``` +fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { + let table: toml::Table = text + .parse() + .map_err(|err| format!("protocol is not valid TOML: {err}"))?; + let loops = match table.get("loops") { + None => 1, + Some(value) => { + let loops = value.as_integer().ok_or("loops must be an integer")?; + if !(1..=10_000).contains(&loops) { + return Err("loops must be between 1 and 10000".into()); + } + loops as usize + } + }; + let raw_steps = table + .get("steps") + .and_then(|value| value.as_array()) + .ok_or("protocol needs at least one [[steps]] entry")?; + if raw_steps.is_empty() { + return Err("protocol needs at least one [[steps]] entry".into()); + } + + let mut steps = Vec::with_capacity(raw_steps.len()); + for (index, raw) in raw_steps.iter().enumerate() { + let step = raw + .as_table() + .ok_or_else(|| format!("step {} must be a table", index + 1))?; + let context = |msg: &str| format!("step {}: {msg}", index + 1); + + let duration_s = step + .get("duration_s") + .and_then(|value| value.as_float().or(value.as_integer().map(|v| v as f64))) + .ok_or_else(|| context("duration_s is required"))?; + if !(0.001..=3_600.0).contains(&duration_s) { + return Err(context("duration_s must be between 0.001 and 3600")); + } + let wave = step + .get("wave") + .and_then(|value| value.as_str()) + .ok_or_else(|| context("wave is required (OFF/CONST/SINE/SQUARE)"))? + .to_uppercase(); + + let (command, summary) = if wave == "OFF" { + ( + Command::new("MOD").field("wave", "OFF"), + format!("OFF for {duration_s} s"), + ) + } else { + let mode = Mode::from_name(&wave) + .ok_or_else(|| context("wave must be OFF, CONST, SINE, or SQUARE"))?; + let level = step + .get("level") + .and_then(|value| value.as_integer()) + .ok_or_else(|| context("level is required"))?; + if !(0..=MAX_DAC_CODE).contains(&level) { + return Err(context("level must be between 0 and 4095")); + } + let mut command = Command::new("MOD") + .field("wave", mode.name()) + .field("level", level); + let summary; + if mode.is_periodic() { + let frequency_hz = step + .get("frequency_hz") + .and_then(|value| value.as_float().or(value.as_integer().map(|v| v as f64))) + .ok_or_else(|| context("frequency_hz is required for SINE/SQUARE"))?; + if !(0.01..=2_000.0).contains(&frequency_hz) { + return Err(context("frequency_hz must be between 0.01 and 2000")); + } + let min = step + .get("min") + .and_then(|value| value.as_integer()) + .unwrap_or(0); + if !(0..=level).contains(&min) { + return Err(context("min must be between 0 and level")); + } + command = command + .field("min", min) + .field("freq_mhz", (frequency_hz * 1_000.0).round() as i64); + summary = format!( + "{} {min}..{level} @ {frequency_hz} Hz for {duration_s} s", + mode.name() + ); + } else { + summary = format!("CONST level={level} for {duration_s} s"); + } + (command, summary) + }; + steps.push(ProtocolStep { + duration: Duration::from_secs_f64(duration_s), + command, + summary, + }); + } + Ok((steps, loops)) +} + +/// Walks the steps on an absolute schedule (no drift accumulation); the last +/// commanded step holds after completion — set-and-hold, like the firmware. +fn run_protocol( + steps: Vec, + loops: usize, + shared: Arc, + stop: Arc, + progress: Arc>, +) { + let mut next_deadline = Instant::now(); + 'run: for loop_index in 1..=loops { + for (step_index, step) in steps.iter().enumerate() { + if stop.load(Ordering::Relaxed) { + break 'run; + } + if let Ok(mut progress) = progress.lock() { + progress.loop_index = loop_index; + progress.step_index = step_index + 1; + progress.summary = step.summary.clone(); + } + *shared.pending.lock().expect("pending lock") = Some(step.command.clone()); + shared.bump(); + next_deadline += step.duration; + while Instant::now() < next_deadline { + if stop.load(Ordering::Relaxed) { + break 'run; + } + let remaining = next_deadline.saturating_duration_since(Instant::now()); + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } + } + } + if let Ok(mut progress) = progress.lock() { + progress.finished = true; + progress.stopped = stop.load(Ordering::Relaxed); + } + shared.bump(); +} + pub struct StageAModulationPlugin { enabled: bool, link: Option, shared: Arc, + protocol: Option, // -- settings (every accepted change is sent immediately) -- connect_requested: bool, port_hint: String, @@ -254,6 +441,7 @@ pub struct StageAModulationPlugin { min_level: i64, mode: Mode, frequency_hz: f64, + protocol_path: String, last_error: Option, } @@ -263,6 +451,7 @@ impl Default for StageAModulationPlugin { enabled: false, link: None, shared: Arc::new(SharedLink::new()), + protocol: None, connect_requested: false, port_hint: "auto".into(), max_level: MAX_DAC_CODE, @@ -270,6 +459,7 @@ impl Default for StageAModulationPlugin { min_level: 0, mode: Mode::Const, frequency_hz: 10.0, + protocol_path: String::new(), last_error: None, } } @@ -328,10 +518,59 @@ impl StageAModulationPlugin { } fn disconnect(&mut self) { + // A protocol without a device to drain its commands is meaningless. + self.protocol = None; self.link = None; // Drop stops and joins the device thread. self.shared.bump(); } + fn protocol_active(&self) -> bool { + self.protocol + .as_ref() + .is_some_and(|run| !run.progress.lock().map(|p| p.finished).unwrap_or(true)) + } + + fn start_protocol(&mut self) -> Result<(), String> { + if self.protocol_active() { + return Ok(()); + } + if self.link.is_none() { + return Err("connect to the controller before running a protocol".into()); + } + if self.protocol_path.trim().is_empty() { + return Err("choose a protocol file first".into()); + } + let text = std::fs::read_to_string(self.protocol_path.trim()) + .map_err(|err| format!("reading {} failed: {err}", self.protocol_path.trim()))?; + let (steps, loops) = parse_protocol(&text)?; + let stop = Arc::new(AtomicBool::new(false)); + let progress = Arc::new(Mutex::new(ProtocolProgress { + loops, + total_steps: steps.len(), + ..ProtocolProgress::default() + })); + let join = std::thread::Builder::new() + .name("stage-a-modulation-protocol".into()) + .spawn({ + let shared = Arc::clone(&self.shared); + let stop = Arc::clone(&stop); + let progress = Arc::clone(&progress); + move || run_protocol(steps, loops, shared, stop, progress) + }) + .expect("spawning the protocol thread must succeed"); + self.protocol = Some(ProtocolRun { + stop, + join: Some(join), + progress, + }); + Ok(()) + } + + fn stop_protocol(&mut self) { + self.protocol = None; // Drop stops and joins; last command holds. + self.shared.bump(); + } + /// Queues one MOD command carrying the complete current drive settings; /// newer changes overwrite queued ones (drag coalescing). fn send_modulation(&mut self) { @@ -571,109 +810,147 @@ impl Plugin for StageAModulationPlugin { .position(|m| *m == self.mode) .unwrap_or(0); SettingsSchema { - sections: vec![SettingsSection { - label: "Laser modulation".into(), - description: Some( - "Tick Connect, then every change is sent to the Teensy immediately — no \ + sections: vec![ + SettingsSection { + label: "Laser modulation".into(), + description: Some( + "Tick Connect, then every change is sent to the Teensy immediately — no \ camera required. The output never exceeds the power slider, the slider \ never exceeds the max limit. The firmware holds the output when \ disconnected; drag the slider to 0 to drive 0 V." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "auto (recommended) probes the attached usbmodem ports and picks \ + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "auto (recommended) probes the attached usbmodem ports and picks \ the one that answers HELLO — the Teensy command port; \ mock = in-process simulated controller" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, }, - }, - SettingItem { - key: "connect".into(), - label: "Connect".into(), - tooltip: Some( - "Opens/closes the command port. Connecting never changes the \ + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the command port. Connecting never changes the \ output; disconnecting leaves it held (set-and-hold firmware)." - .into(), - ), - kind: SettingKind::Bool { - default: self.connect_requested, + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, }, - }, - SettingItem { - key: "level".into(), - label: "Power (DAC code)".into(), - tooltip: Some( - "Output level in DAC codes; peak value for sine/square. \ + SettingItem { + key: "level".into(), + label: "Power (DAC code)".into(), + tooltip: Some( + "Output level in DAC codes; peak value for sine/square. \ Capped by the max limit below. 0 = output off." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.level, - suffix: None, + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.level, + suffix: None, + }, }, - }, - SettingItem { - key: "max_level".into(), - label: "Max limit (DAC code)".into(), - tooltip: Some( - "Safety cap: the slider cannot go above this. Set it to the \ + SettingItem { + key: "max_level".into(), + label: "Max limit (DAC code)".into(), + tooltip: Some( + "Safety cap: the slider cannot go above this. Set it to the \ highest code the connected device tolerates at J23." - .into(), - ), - kind: SettingKind::I64Drag { - min: 0, - max: MAX_DAC_CODE, - default: self.max_level, + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.max_level, + }, }, - }, - SettingItem { - key: "mode".into(), - label: "Mode".into(), - tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), - kind: SettingKind::Enum { - variants: mode_variants, - default: mode_default, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, }, - }, - SettingItem { - key: "frequency_hz".into(), - label: "Frequency".into(), - tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), - kind: SettingKind::F64Drag { - min: 0.01, - max: 2_000.0, - speed: 1.0, - default: self.frequency_hz, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.frequency_hz, + }, }, - }, - SettingItem { - key: "min_level".into(), - label: "Min threshold (DAC code)".into(), - tooltip: Some( - "Lower bound for sine/square: the waveform swings between this \ + SettingItem { + key: "min_level".into(), + label: "Min threshold (DAC code)".into(), + tooltip: Some( + "Lower bound for sine/square: the waveform swings between this \ and the power slider. Ignored in CONST mode." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.min_level, - suffix: None, + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.min_level, + suffix: None, + }, }, - }, - ], - }], + ], + }, + SettingsSection { + label: "Protocol".into(), + description: Some( + "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ + [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ + min, frequency_hz. Steps run on an absolute schedule; the last \ + step holds after completion (set-and-hold). Stopping never \ + switches the output off by itself." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "protocol_path".into(), + label: "Protocol file".into(), + tooltip: Some("TOML protocol file (validated on start).".into()), + kind: SettingKind::Path { + dialog: PathDialogKind::OpenFile, + default: self.protocol_path.clone(), + }, + }, + SettingItem { + key: "protocol_run".into(), + label: "Run protocol".into(), + tooltip: Some( + "Start/stop the loaded protocol. Requires an open connection; \ + manual drive controls stay live and override the current step \ + until the next one begins." + .into(), + ), + kind: SettingKind::Bool { + default: self.protocol_active(), + }, + }, + ], + }, + ], } } @@ -700,6 +977,8 @@ impl Plugin for StageAModulationPlugin { } "frequency_hz" => Some(json!(self.frequency_hz)), "min_level" => Some(json!(self.min_level)), + "protocol_path" => Some(json!(self.protocol_path)), + "protocol_run" => Some(json!(self.protocol_active())), _ => None, } } @@ -773,6 +1052,27 @@ impl Plugin for StageAModulationPlugin { } Ok(()) } + "protocol_path" => { + self.protocol_path = value + .as_str() + .ok_or("protocol_path must be a string")? + .to_owned(); + Ok(()) + } + "protocol_run" => { + let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; + // Failures surface through status entries (like `connect`). + if requested { + match self.start_protocol() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + } else { + self.stop_protocol(); + } + self.shared.bump(); + Ok(()) + } _ => Err(format!("unknown setting: {key}")), } } @@ -793,6 +1093,26 @@ impl Plugin for StageAModulationPlugin { state.board_mod ))); } + if let Some(run) = &self.protocol { + if let Ok(progress) = run.progress.lock() { + entries.push(StatusEntry::Text(if progress.finished { + if progress.stopped { + "Protocol: stopped (last step holds)".into() + } else { + "Protocol: finished (last step holds)".into() + } + } else { + format!( + "Protocol: loop {}/{} step {}/{} — {}", + progress.loop_index, + progress.loops, + progress.step_index, + progress.total_steps, + progress.summary + ) + })); + } + } if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { entries.push(StatusEntry::Text(format!("Error: {error}"))); } @@ -944,6 +1264,123 @@ mod tests { plugin.set_setting("connect", json!(false)).unwrap(); } + const TEST_PROTOCOL: &str = r#" +loops = 2 + +[[steps]] +duration_s = 0.03 +wave = "SINE" +level = 2000 +min = 100 +frequency_hz = 100.0 + +[[steps]] +duration_s = 0.03 +wave = "CONST" +level = 750 +"#; + + #[test] + fn protocol_parsing_validates_steps() { + let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); + assert_eq!(loops, 2); + assert_eq!(steps.len(), 2); + let encoded = |command: &Command, seq: u32| { + String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") + }; + assert_eq!( + encoded(&steps[0].command, 1), + "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" + ); + assert_eq!( + encoded(&steps[1].command, 2), + "@2 MOD wave=CONST level=750\n" + ); + assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); + + assert!(parse_protocol("loops = 1").is_err(), "steps required"); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), + "periodic steps need a frequency" + ); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), + "level range enforced" + ); + assert!( + parse_protocol( + "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" + ) + .is_err(), + "min above level rejected" + ); + let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") + .expect("OFF needs no level"); + assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); + } + + /// A protocol against the mock walks every step, holds the last one, and + /// reports finished. + #[test] + fn protocol_runs_to_completion_on_the_mock() { + let dir = std::env::temp_dir().join(format!( + "stage-a-modulation-protocol-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("protocol.toml"); + std::fs::write(&path, TEST_PROTOCOL).unwrap(); + + let mut plugin = StageAModulationPlugin::default(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + + plugin + .set_setting("protocol_path", json!(path.display().to_string())) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); + + // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. + wait_until(&plugin, Duration::from_secs(3), |p| { + !p.protocol_active() && board_code(p) == Some(750) + }); + assert!(!plugin.protocol_active()); + assert_eq!(board_code(&plugin), Some(750), "last step holds"); + let progress = plugin + .protocol + .as_ref() + .unwrap() + .progress + .lock() + .unwrap() + .clone(); + assert!(progress.finished && !progress.stopped); + assert_eq!((progress.loop_index, progress.step_index), (2, 2)); + + plugin.set_setting("connect", json!(false)).unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn protocol_requires_a_connection() { + let mut plugin = StageAModulationPlugin::default(); + plugin + .set_setting("protocol_path", json!("/tmp/x.toml")) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("connect"))); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); + } + /// The host settings UI exchanges enum values as indices into the /// schema's variant list (radio buttons send `json!(index)`). #[test] From 38752e3a2ae50a715abc59a32bda6a6777594c1f Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Fri, 17 Jul 2026 17:14:03 +0200 Subject: [PATCH 19/30] =?UTF-8?q?perf(stage-a):=20=E2=9A=A1=20decimate=20t?= =?UTF-8?q?he=20photodiode=20chart=20from=20incremental=20summary=20cells?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepares the plugin for firmware 0.5.0's 500 kSa/s DMA stream (ADR 004 in stage-a-controller): a full raw-window rescan per repaint stops being viable around that rate. - ingest maintains 64:1 min/max/sum summary cells aligned to deque offsets; eviction drops whole cells so the alignment (and the device-clock index base) survives, at the cost of up to one cell of ring slack - chart buckets and every moving-average window combine cells plus raw edge samples via range_summary — O(range/64) instead of O(range), verified exact against naive scans across cell boundaries and after eviction - ring cap raised to 16 M samples (32 s at 500 kSa/s, 32 MiB of codes); cache_s keeps ruling the duration at lower rates Verified: cargo fmt, clippy -D warnings, 16 plugin tests green. --- plugins/stage-a-photodiode/src/lib.rs | 294 ++++++++++++++++++++------ 1 file changed, 224 insertions(+), 70 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index ffc2d48..80afa6d 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -54,9 +54,12 @@ const ADC_MAX_CODE: f64 = 4_095.0; /// (user-settable 1–130 s). const DEFAULT_CACHE_SECONDS: f64 = 20.0; const MAX_CACHE_SECONDS: f64 = 130.0; -/// Absolute sample cap guarding against absurd advertised rates (8 MiB of -/// codes at most). -const RING_MAX_SAMPLES: usize = 4_000_000; +/// Absolute sample cap: 16 M samples = 32 s at the firmware's 500 kSa/s +/// stream rate (32 MiB of codes + ~2 MiB of summary cells). +const RING_MAX_SAMPLES: usize = 16_000_000; +/// Raw samples per incremental summary cell (min/max/sum), the unit both +/// chart decimation and the moving average combine instead of raw rescans. +const SUMMARY_CELL: usize = 64; /// Envelope buckets per rendered chart line; keeps the plot payload bounded /// no matter how many raw samples the window covers. const MAX_PLOT_BUCKETS: usize = 1_000; @@ -130,6 +133,11 @@ struct SharedState { /// Device sample index of `samples.front()` within the current segment. ring_first_index: u64, samples: VecDeque, + /// Incremental 64:1 summaries: `cells[i]` covers deque offsets + /// `[i·CELL, (i+1)·CELL)`. Kept aligned by evicting whole cells, so the + /// chart and moving average never rescan the raw window — at 500 kSa/s a + /// full-window rescan per repaint would not be viable. + cells: VecDeque, latest: Option, /// Cumulative firmware-side drop counter (latest header value). device_dropped: u32, @@ -142,12 +150,39 @@ struct SharedState { error: Option, } +/// min/max/sum over exactly [`SUMMARY_CELL`] consecutive raw samples. +#[derive(Clone, Copy)] +struct SummaryCell { + min: u16, + max: u16, + sum: u32, +} + +/// Accumulated min/max/sum/count over an arbitrary sample range. +#[derive(Clone, Copy)] +struct RangeSummary { + min: u16, + max: u16, + sum: u64, + count: usize, +} + +impl RangeSummary { + fn mean(&self) -> f64 { + if self.count == 0 { + return 0.0; + } + self.sum as f64 / self.count as f64 + } +} + impl Default for SharedState { fn default() -> Self { Self { rate_hz: 0, ring_first_index: 0, samples: VecDeque::new(), + cells: VecDeque::new(), latest: None, device_dropped: 0, crc_failures: 0, @@ -179,20 +214,90 @@ impl SharedState { self.segments += 1; } self.samples.clear(); + self.cells.clear(); self.ring_first_index = first_index; self.rate_hz = rate_hz; } self.samples.extend(codes.iter().copied()); self.latest = codes.last().copied(); self.device_dropped = device_dropped; + + // Summarize every newly completed cell. + while (self.cells.len() + 1) * SUMMARY_CELL <= self.samples.len() { + let start = self.cells.len() * SUMMARY_CELL; + let mut cell = SummaryCell { + min: u16::MAX, + max: u16::MIN, + sum: 0, + }; + for &code in self.samples.range(start..start + SUMMARY_CELL) { + cell.min = cell.min.min(code); + cell.max = cell.max.max(code); + cell.sum += u32::from(code); + } + self.cells.push_back(cell); + } + + // Evict whole cells only, keeping the cell/offset alignment intact; + // the ring may exceed its capacity by up to one cell. let excess = self .samples .len() .saturating_sub(self.ring_capacity(rate_hz)); - if excess > 0 { - self.samples.drain(..excess); - self.ring_first_index += excess as u64; + let evict_cells = excess / SUMMARY_CELL; + if evict_cells > 0 { + let evict = evict_cells * SUMMARY_CELL; + self.samples.drain(..evict); + self.cells.drain(..evict_cells); + self.ring_first_index += evict as u64; + } + } + + /// min/max/sum over deque offsets `[start, end)`, combining whole + /// summary cells with raw samples at the edges: O(range/64 + 128) + /// instead of O(range). + fn range_summary(&self, start: usize, end: usize) -> RangeSummary { + let end = end.min(self.samples.len()); + let mut summary = RangeSummary { + min: u16::MAX, + max: u16::MIN, + sum: 0, + count: 0, + }; + if start >= end { + return summary; + } + summary.count = end - start; + let covered = self.cells.len() * SUMMARY_CELL; + let mut i = start; + + // Raw head up to the next cell boundary. + let head_end = (i.div_ceil(SUMMARY_CELL) * SUMMARY_CELL) + .min(end) + .min(covered.max(i)); + if head_end > i { + for &code in self.samples.range(i..head_end) { + summary.min = summary.min.min(code); + summary.max = summary.max.max(code); + summary.sum += u64::from(code); + } + i = head_end; + } + // Whole cells. + while i + SUMMARY_CELL <= end.min(covered) { + let cell = self.cells[i / SUMMARY_CELL]; + summary.min = summary.min.min(cell.min); + summary.max = summary.max.max(cell.max); + summary.sum += u64::from(cell.sum); + i += SUMMARY_CELL; } + // Raw tail (past the last whole cell in range, or past `covered`). + for &code in self.samples.range(i..end) { + summary.min = summary.min.min(code); + summary.max = summary.max.max(code); + summary.sum += u64::from(code); + } + summary } } @@ -736,8 +841,7 @@ impl StageAPhotodiodePlugin { .avg_window_samples(state.rate_hz) .min(state.samples.len()); let start = state.samples.len() - window; - let sum: u64 = state.samples.range(start..).map(|&c| u64::from(c)).sum(); - Some(sum as f64 / window as f64) + Some(state.range_summary(start, state.samples.len()).mean()) } fn series_dataset(&self) -> Series1dV1 { @@ -777,76 +881,55 @@ impl StageAPhotodiodePlugin { let avg_window = self.avg_window_samples(state.rate_hz); let avg_enabled = avg_window > 1; - // Prime the running sum with up to `avg_window − 1` samples that - // precede the visible slice, so the average is correct from the - // first visible point on. - let prime_start = start.saturating_sub(avg_window - 1); - let mut avg_sum: u64 = 0; - let mut avg_count: usize = 0; - for &code in state.samples.range(prime_start..start) { - avg_sum += u64::from(code); - avg_count += 1; - } let mut mean_points = Vec::with_capacity(MAX_PLOT_BUCKETS + 1); let mut min_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); let mut max_points = Vec::with_capacity(if decimating { MAX_PLOT_BUCKETS + 1 } else { 0 }); let mut avg_points = Vec::with_capacity(if avg_enabled { MAX_PLOT_BUCKETS + 1 } else { 0 }); - let mut bucket_min = u16::MAX; - let mut bucket_max = u16::MIN; - let mut bucket_sum: u64 = 0; - let mut bucket_n: usize = 0; - for (offset, &code) in state.samples.range(start..).enumerate() { - let i = start + offset; - bucket_min = bucket_min.min(code); - bucket_max = bucket_max.max(code); - bucket_sum += u64::from(code); - bucket_n += 1; - if avg_enabled { - avg_sum += u64::from(code); - avg_count += 1; - if avg_count > avg_window { - avg_sum -= u64::from(state.samples[i - avg_window]); - avg_count -= 1; - } + // Every bucket (and every moving-average window) is combined from + // the incremental summary cells plus raw edge samples — the cost per + // rebuild is O(buckets · window/64), independent of the raw rate. + let mut bucket_start = start; + while bucket_start < total { + let bucket_end = (bucket_start + bucket_len).min(total); + let last = bucket_end - 1; + let bucket = state.range_summary(bucket_start, bucket_end); + let device_t = (state.ring_first_index + last as u64) as f64 / rate; + let x = match self.time_axis { + TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, + TimeAxis::Segment => device_t, + }; + mean_points.push(Series1dPoint { + x, + y: self.display_volts(bucket.mean()), + }); + if decimating { + // EXCITATION inverts the axis, so min/max swap roles. + let (low, high) = ( + self.display_volts(f64::from(bucket.min)), + self.display_volts(f64::from(bucket.max)), + ); + min_points.push(Series1dPoint { + x, + y: low.min(high), + }); + max_points.push(Series1dPoint { + x, + y: low.max(high), + }); } - if bucket_n == bucket_len || i == total - 1 { - let device_t = (state.ring_first_index + i as u64) as f64 / rate; - let x = match self.time_axis { - TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, - TimeAxis::Segment => device_t, - }; - mean_points.push(Series1dPoint { + if avg_enabled { + // Trailing window ending at this bucket's last sample; may + // reach before the visible slice (fewer while filling). + let window_start = (last + 1).saturating_sub(avg_window); + let window = state.range_summary(window_start, last + 1); + avg_points.push(Series1dPoint { x, - y: self.display_volts(bucket_sum as f64 / bucket_n as f64), + y: self.display_volts(window.mean()), }); - if decimating { - // EXCITATION inverts the axis, so min/max swap roles. - let (low, high) = ( - self.display_volts(f64::from(bucket_min)), - self.display_volts(f64::from(bucket_max)), - ); - min_points.push(Series1dPoint { - x, - y: low.min(high), - }); - max_points.push(Series1dPoint { - x, - y: low.max(high), - }); - } - if avg_enabled { - avg_points.push(Series1dPoint { - x, - y: self.display_volts(avg_sum as f64 / avg_count as f64), - }); - } - bucket_min = u16::MAX; - bucket_max = u16::MIN; - bucket_sum = 0; - bucket_n = 0; } + bucket_start = bucket_end; } let mut lines = vec![Series1dLine { @@ -1835,12 +1918,22 @@ mod tests { state.ingest(index, rate, 0, &block); index += block.len() as u64; } - assert_eq!(state.samples.len(), cap); + // Whole-cell eviction may leave up to one summary cell of slack. + assert!( + state.samples.len() >= cap && state.samples.len() < cap + SUMMARY_CELL, + "len {} vs cap {cap}", + state.samples.len() + ); assert_eq!( state.ring_first_index + state.samples.len() as u64, index, "eviction keeps indexes aligned" ); + assert_eq!( + state.ring_first_index % SUMMARY_CELL as u64, + 0, + "eviction preserves cell alignment" + ); assert_eq!(state.segments, 0, "eviction is not a discontinuity"); } @@ -1957,6 +2050,62 @@ mod tests { plugin.disconnect(); } + /// The summary cells must agree exactly with a naive raw scan for + /// arbitrary ranges, including after whole-cell eviction. + #[test] + fn range_summary_matches_naive_scans() { + let mut state = SharedState { + cache_seconds: 1.0, // capacity 1000 at rate 1000 → forces eviction + ..SharedState::default() + }; + let mut hash: u64 = 0x243F_6A88_85A3_08D3; + let mut next = || { + hash ^= hash << 13; + hash ^= hash >> 7; + hash ^= hash << 17; + (hash % 4_096) as u16 + }; + let mut index = 0_u64; + for _ in 0..7 { + let block: Vec = (0..333).map(|_| next()).collect(); + state.ingest(index, 1_000, 0, &block); + index += block.len() as u64; + } + assert!(state.samples.len() <= 1_000 + SUMMARY_CELL, "evicted"); + assert!(!state.cells.is_empty()); + + let len = state.samples.len(); + for (start, end) in [ + (0, len), + (0, 1), + (1, SUMMARY_CELL), + (SUMMARY_CELL - 1, SUMMARY_CELL + 1), + (7, 500), + (130, 131), + (len - 3, len), + (len / 3, 2 * len / 3), + ] { + let summary = state.range_summary(start, end); + let raw: Vec = state.samples.range(start..end).copied().collect(); + assert_eq!(summary.count, raw.len(), "count for {start}..{end}"); + assert_eq!( + summary.min, + raw.iter().copied().min().unwrap(), + "min for {start}..{end}" + ); + assert_eq!( + summary.max, + raw.iter().copied().max().unwrap(), + "max for {start}..{end}" + ); + assert_eq!( + summary.sum, + raw.iter().map(|&c| u64::from(c)).sum::(), + "sum for {start}..{end}" + ); + } + } + #[test] fn spectrum_finds_a_synthesized_tone() { let plugin = StageAPhotodiodePlugin::default(); @@ -2126,7 +2275,12 @@ mod tests { let first = i * 1_000; state.ingest(first, 1_000, 0, &block); } - assert_eq!(state.samples.len(), 2_000); + // Whole-cell eviction may leave up to one summary cell of slack. + assert!( + state.samples.len() >= 2_000 && state.samples.len() < 2_000 + SUMMARY_CELL, + "len {}", + state.samples.len() + ); } /// The host settings UI exchanges enum values as indices into the From bb83705ae15a5c8e65d6953be0fe42c667eefa72 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 20 Jul 2026 11:32:10 +0200 Subject: [PATCH 20/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20report=20t?= =?UTF-8?q?he=20legacy=20ASCII=20stream=20as=20a=20firmware-flash=20hint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The photodiode plugin dropped the pre-0.4.0 ASCII PD-line path, so a Teensy running old firmware produced a generic 'no PDA1 sample frames' error that gave no hint at the real cause. The auto-probe now classifies each port (PDA1 frames / legacy ASCII / nothing) and, when it sees the 'PD code=…' ASCII stream, tells the user to flash stage-a-controller 0.4.0+ instead — the actual fix, since the plugin and firmware ship in lockstep. Verified: cargo fmt, clippy -D warnings, 16 tests green. --- plugins/stage-a-photodiode/src/lib.rs | 57 +++++++++++++++++++++------ 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 80afa6d..1e0cbff 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -1201,10 +1201,25 @@ fn resolve_auto_port() -> Result { if candidates.is_empty() { return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); } + let mut saw_legacy_ascii = false; for path in &candidates { - if probe_pd_stream(path) { - return Ok(path.clone()); - } + match probe_pd_stream(path) { + ProbeResult::Pda1SampleFrames => return Ok(path.clone()), + ProbeResult::LegacyAsciiStream => saw_legacy_ascii = true, + ProbeResult::Nothing => {} + } + } + if saw_legacy_ascii { + // The pre-0.4.0 firmware emits `PD code=… n=… t_ms=…` ASCII lines + // instead of PDA1 binary frames. This plugin dropped the ASCII path + // (three-repo lockstep), so the fix is a firmware flash, not a plugin + // setting — say so instead of a generic "no frames". + return Err(format!( + "found the legacy ASCII photodiode stream (pre-0.4.0 firmware) — flash \ + stage-a-controller 0.4.0+ so the stream port emits PDA1 binary frames \ + (tried {})", + candidates.join(", ") + )); } Err(format!( "no port streamed PDA1 sample frames within 500 ms (tried {})", @@ -1212,18 +1227,29 @@ fn resolve_auto_port() -> Result { )) } -/// True when `path` produces a CRC-clean `SamplesU16` frame within the probe -/// window. The command port emits frames too, but only control replies and -/// acquisition data — unsolicited sample frames identify the stream port. -fn probe_pd_stream(path: &str) -> bool { +/// What a brief listen on a candidate port revealed. +enum ProbeResult { + /// CRC-clean PDA1 `SamplesU16` frames — the 0.4.0+ stream port. + Pda1SampleFrames, + /// `PD code=… n=… t_ms=…` ASCII lines — the pre-0.4.0 stream port. + LegacyAsciiStream, + /// Nothing parsable (busy/command port, wrong device, or no data). + Nothing, +} + +/// Listens on `path` for up to 500 ms and classifies what it emits. The +/// command port emits frames too, but only control replies and acquisition +/// data — unsolicited sample frames identify the stream port. +fn probe_pd_stream(path: &str) -> ProbeResult { let Ok(mut port) = serialport::new(path, 115_200) .timeout(Duration::from_millis(100)) .open() else { - return false; + return ProbeResult::Nothing; }; let deadline = Instant::now() + Duration::from_millis(500); let mut parser = FrameParser::default(); + let mut ascii_tail: Vec = Vec::with_capacity(256); let mut buf = [0_u8; 4_096]; while Instant::now() < deadline { match port.read(&mut buf) { @@ -1232,19 +1258,28 @@ fn probe_pd_stream(path: &str) -> bool { while let Some(event) = parser.next_event() { if let ParseEvent::Frame(frame) = event { if frame.samples().is_some() { - return true; + return ProbeResult::Pda1SampleFrames; } } } + // Sniff for the legacy ASCII line format in parallel; a valid + // `PD code=` prefix never appears inside PDA1 binary framing. + ascii_tail.extend_from_slice(&buf[..read]); + if String::from_utf8_lossy(&ascii_tail).contains("PD code=") { + return ProbeResult::LegacyAsciiStream; + } + if ascii_tail.len() > 512 { + ascii_tail.drain(..ascii_tail.len() - 256); + } } Ok(_) => {} Err(err) if err.kind() == std::io::ErrorKind::TimedOut || err.kind() == std::io::ErrorKind::Interrupted => {} - Err(_) => return false, + Err(_) => return ProbeResult::Nothing, } } - false + ProbeResult::Nothing } /// The exact variant list the settings schema shows for the port enum — the From 4046b7aac09204c0a1fce4a59b6e268444a0157e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 19:49:11 +0200 Subject: [PATCH 21/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20add=20the=20?= =?UTF-8?q?A1=20orchestration=20plugin=20and=20shared=20plugin=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces `stage-a-a1` as the measurement owner that drives the modulation and photodiode plugins through a leased device contract, plus the `stage-a-plugin-contract` crate that carries the shared settings/telemetry schema between them. Also lands the supporting work these depend on: - optical waveform generation and inversion in the modulation plugin - Pockels transfer calibration (measured V_null / Vpi) with sweep support - photodiode contrast estimation and monitor-cache snapshots - `.pdq` recording format with SHA-256 integrity sidecars Documented in docs/features/stage-a-a1.md, stage-a-a1-automation.md, stage-a-optical-waveform.md, stage-a-pockels-calibration.md and ADRs 007-011. --- Cargo.toml | 3 + README.md | 3 + docs/adr/005-stage-a-device-ownership.md | 14 +- docs/adr/006-stage-a-two-plugin-split.md | 20 +- docs/adr/007-stage-a-owner-orchestration.md | 68 + .../008-stage-a-optical-waveform-inversion.md | 67 + .../009-stage-a-a1-recording-coordinator.md | 107 + docs/adr/010-stage-a-a1-amplitude-sweep.md | 72 + ...11-stage-a-pockels-transfer-calibration.md | 151 + docs/architecture.md | 22 + docs/features/README.md | 8 +- docs/features/stage-a-a1-automation.md | 110 + docs/features/stage-a-a1.md | 204 + docs/features/stage-a-modulation.md | 64 +- docs/features/stage-a-optical-waveform.md | 125 + docs/features/stage-a-photodiode.md | 32 +- docs/features/stage-a-pockels-calibration.md | 214 + docs/features/stage-a.md | 26 +- plugins/stage-a-a1/Cargo.toml | 20 + plugins/stage-a-a1/README.md | 53 + plugins/stage-a-a1/plugin.toml | 9 + plugins/stage-a-a1/src/lib.rs | 14 + plugins/stage-a-a1/src/phase.rs | 382 ++ plugins/stage-a-a1/src/rates.rs | 294 + plugins/stage-a-a1/src/response_curve.rs | 293 + plugins/stage-a-a1/src/runtime.rs | 3460 +++++++++++ plugins/stage-a-a1/src/types.rs | 26 + plugins/stage-a-modulation/Cargo.toml | 1 + plugins/stage-a-modulation/README.md | 85 +- plugins/stage-a-modulation/plugin.toml | 5 +- plugins/stage-a-modulation/src/calibration.rs | 786 +++ plugins/stage-a-modulation/src/lib.rs | 5096 ++++++++++++++--- plugins/stage-a-modulation/src/waveform.rs | 389 ++ plugins/stage-a-photodiode/Cargo.toml | 1 + plugins/stage-a-photodiode/README.md | 30 +- plugins/stage-a-photodiode/plugin.toml | 1 + plugins/stage-a-photodiode/src/lib.rs | 1716 +++++- stage-a-io/src/estimator.rs | 169 +- stage-a-io/src/lib.rs | 18 +- stage-a-io/src/mock.rs | 101 + stage-a-io/src/pdq.rs | 512 +- stage-a-io/src/sha256.rs | 259 + stage-a-io/src/sidecar.rs | 10 + stage-a-io/src/wire.rs | 87 +- stage-a-plugin-contract/Cargo.toml | 17 + stage-a-plugin-contract/README.md | 47 + stage-a-plugin-contract/src/lib.rs | 853 +++ 47 files changed, 15110 insertions(+), 934 deletions(-) create mode 100644 docs/adr/007-stage-a-owner-orchestration.md create mode 100644 docs/adr/008-stage-a-optical-waveform-inversion.md create mode 100644 docs/adr/009-stage-a-a1-recording-coordinator.md create mode 100644 docs/adr/010-stage-a-a1-amplitude-sweep.md create mode 100644 docs/adr/011-stage-a-pockels-transfer-calibration.md create mode 100644 docs/features/stage-a-a1-automation.md create mode 100644 docs/features/stage-a-a1.md create mode 100644 docs/features/stage-a-optical-waveform.md create mode 100644 docs/features/stage-a-pockels-calibration.md create mode 100644 plugins/stage-a-a1/Cargo.toml create mode 100644 plugins/stage-a-a1/README.md create mode 100644 plugins/stage-a-a1/plugin.toml create mode 100644 plugins/stage-a-a1/src/lib.rs create mode 100644 plugins/stage-a-a1/src/phase.rs create mode 100644 plugins/stage-a-a1/src/rates.rs create mode 100644 plugins/stage-a-a1/src/response_curve.rs create mode 100644 plugins/stage-a-a1/src/runtime.rs create mode 100644 plugins/stage-a-a1/src/types.rs create mode 100644 plugins/stage-a-modulation/src/calibration.rs create mode 100644 plugins/stage-a-modulation/src/waveform.rs create mode 100644 stage-a-io/src/sha256.rs create mode 100644 stage-a-plugin-contract/Cargo.toml create mode 100644 stage-a-plugin-contract/README.md create mode 100644 stage-a-plugin-contract/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index dbee199..abb6328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ "stage-a-io", + "stage-a-plugin-contract", + "plugins/stage-a-a1", "plugins/stage-a-modulation", "plugins/stage-a-photodiode", "plugins/localization", @@ -29,3 +31,4 @@ rustfft = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" serialport = "4" +stage-a-plugin-contract = { path = "stage-a-plugin-contract" } diff --git a/README.md b/README.md index 4d48dd1..346be75 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ The plugin crates under `plugins/` are under active development and not yet read | `evesmlm-candidates` | `RawEvents` | Event-domain candidate clustering plus accepted/rejected raw-event investigation layers | | `evesmlm-fitting` | `DerivedData` | Candidate fitting plus shared current-localization datasets, stable ids, and linked 3D inspection | | `evesmlm-postproc` | `DerivedData` | Filtering, drift correction, evaluation, and the later shared EVE current-localization provider | +| `stage-a-modulation` | control service | Sole owner of the Stage-A Teensy command port and ACKed modulation state | +| `stage-a-photodiode` | control service | Sole owner of the Stage-A stream port, PDA1 ingestion, and PDQ persistence | +| `stage-a-a1` | `RawEvents` + orchestration | A1 protocol/schedule validation, raw phase quicklooks, analysis core, and a safety-gated commissioning run through the two owner services | `plugin-template/` is the starting point for new plugin crates. diff --git a/docs/adr/005-stage-a-device-ownership.md b/docs/adr/005-stage-a-device-ownership.md index 57bcec5..fe7588b 100644 --- a/docs/adr/005-stage-a-device-ownership.md +++ b/docs/adr/005-stage-a-device-ownership.md @@ -2,6 +2,7 @@ - **Status:** Accepted - **Date:** 2026-07-13 +- **Amended by:** ADR 006 and ADR 007 ## Context @@ -14,10 +15,11 @@ laboratory-instrument abstractions. ## Decision -1. **Device control lives in removable protocol plugins** (`stage-a-monitor`, - `stage-a-a1`, later `-a2`/`-a3`), one experiment concern per plugin. - Exactly one enabled, armed plugin owns the serial port; opening a busy - device is a visible error. +1. **Device control lives in removable protocol plugins.** The permanent + command-port and stream-port owners are now `stage-a-modulation` and + `stage-a-photodiode` (ADR 006/007). Experiment workflows such as A1/A2/A3 + orchestrate those owners through the host service plane and do not open the + ports themselves. 2. **A shared plain-Rust library `stage-a-io`** (this repo, not a plugin) owns everything protocol-shaped: PDA1 framing + CRC resync, the ASCII command grammar with idempotent sequence retries, the bounded I/O @@ -35,8 +37,8 @@ laboratory-instrument abstractions. ## Consequences -- A2/A3 plugins reuse `stage-a-io` unchanged; only their state machines - and views are new code. +- A1/A2/A3 reuse the owner services and serde-only contracts; they may reuse + hardware-free `stage-a-io` parsing/analysis but not its serial transports. - The GUI knows nothing about Teensys; removing the three plugins removes every trace of lab hardware from the product. - Protocol changes must land in the firmware header first, then in diff --git a/docs/adr/006-stage-a-two-plugin-split.md b/docs/adr/006-stage-a-two-plugin-split.md index 618e348..1ba4f93 100644 --- a/docs/adr/006-stage-a-two-plugin-split.md +++ b/docs/adr/006-stage-a-two-plugin-split.md @@ -3,6 +3,7 @@ - **Status:** Accepted - **Date:** 2026-07-15 - **Amends:** ADR 005 (Stage-A device ownership) +- **Amended by:** ADR 007 (persistent owners with host-routed orchestration) ## Context @@ -21,30 +22,31 @@ control plugin's connection. ## Decision 1. **The firmware enumerates two USB CDC ports** (`USB_DUAL_SERIAL`, `stage-a-controller` - ADR 002): port 1 keeps the v1 command protocol; port 2 free-runs a plain-ASCII photodiode + ADR 002): port 1 keeps the v1 command protocol; port 2 free-runs the PDA1 photodiode stream. ADR 005's rule is unchanged — one owner per port — there are simply two ports now. 2. **Two minimal plugins replace the three commissioning plugins** (deleted 2026-07-15, retained in git history): - `stage-a-modulation` owns the command port (`docs/features/stage-a-modulation.md`); - `stage-a-photodiode` owns the stream port (`docs/features/stage-a-photodiode.md`). 3. **`stage-a-io` stays** as the protocol library (wire format, client, worker, firmware-faithful - mock — the mock now models firmware 0.3.0's `MOD` verb). The A1/A2/A3 experiment plugins will - build on it again when the bench reaches that stage; the estimator/pdq/sidecar modules remain - for that purpose even though no current plugin uses them. + mock — the mock now models firmware 0.3.0's `MOD` verb). Owner plugins use its transport/PDQ + pieces. A1/A2/A3 do not open transports; they use the host-routed owner contract (ADR 007) + and may use hardware-free parsing/analysis helpers. 4. **Immediate transfer replaces the Apply-action pattern**, and **all device control is settings-driven** (connect checkbox, slider changes sent as they happen). Host actions and the per-frame effects gate are unsuitable here: the host only runs `process_frame()` while camera frames flow, but the bench must work with no camera attached (amended 2026-07-16). Replay mode still disconnects the modulation plugin defensively. The firmware output is - set-and-hold; the power slider at 0 is the off switch. + set-and-hold. ADR 008's 2026-07-23 amendment separates Manual/Calibrated drive method from + waveform mode and makes `max_level` the universal DAC ceiling. ## Consequences -- Each plugin is a few hundred transparent lines with a single concern; the photodiode plugin - does not even depend on `stage-a-io`. +- Each owner plugin has a single hardware concern; the photodiode plugin uses + `stage-a-io`'s PDA1 parser and PDQ persistence without taking command-port ownership. - Both plugins work independently — either can connect, disconnect, or crash without affecting the other. - Wire-protocol changes still land firmware-first (`stage-a-controller/include/wire_protocol.h` and command grammar), then in `stage-a-io`'s client/mock. -- The A1 min-depth workflow is gone from the tree until it is rebuilt on the simplified stack; - its last state is tagged by the deletion commit. +- The A1 min-depth workflow is rebuilt as an orchestrator on this stable two-owner + stack; it never becomes a third Teensy owner. diff --git a/docs/adr/007-stage-a-owner-orchestration.md b/docs/adr/007-stage-a-owner-orchestration.md new file mode 100644 index 0000000..add704a --- /dev/null +++ b/docs/adr/007-stage-a-owner-orchestration.md @@ -0,0 +1,68 @@ +# ADR 007 — Persistent Stage-A owners with host-routed orchestration + +- **Status:** Superseded in part (2026-07-20) — the persistent two-owner model and + host-routed control plane still hold, but `stage-a-a1` no longer orchestrates the + A1 acquisition. It was reduced to a read-only live-analysis plugin (two + phase-folded quicklooks + the photodiode-measured `a`); leases, recordings, + protocol/schedule freezing, references/epochs, and the minimum-depth logistic fit + were removed. See [Stage-A A1 Analysis](../features/stage-a-a1.md). +- **Date:** 2026-07-20 +- **Amends:** ADR 005 and ADR 006 + +## Context + +The A1 workflow must coordinate laser modulation, high-rate photodiode capture, +and camera recording. The earlier handoff proposed that `stage-a-a1` open both +Teensy ports while armed. That would create a third hardware owner and duplicate +the control/readout logic already maintained by the two manual plugins. + +The existing Augur frame context cannot solve this safely: it is available only +inside `process_frame`, while device control and reference acquisition must also +progress without camera frames. Augur also loads a GUI mirror and a live-worker +instance of each plugin, so an effectful setting copied between both instances +can make them compete for the same port. + +## Decision + +1. `stage-a-modulation` is permanently the sole command-port owner and source of + truth for requested and controller-ACKed modulation state. +2. `stage-a-photodiode` is permanently the sole stream-port owner and source of + truth for PDA1 ingestion, integrity accounting, and PDQ persistence. +3. `stage-a-a1` is an orchestrator and camera-analysis plugin. It never opens a + Teensy port and never creates a `PdqWriter`. +4. Coordination uses Augur's frame-independent, worker-owned plugin service + plane. Requests are atomic semantic operations with stable plugin IDs, + request IDs, leases, run IDs, expected revisions, explicit success/rejection, + and bounded versioned snapshots. The host routes messages but contains no + Stage-A logic. +5. Manual controls and automation share the same owner-side validation. A held + automation lease prevents competing manual mutations; a deliberate manual + override revokes the lease, becomes a visible workflow fault, and commands + output-off where safe. +6. Camera start/finalize uses the allow-listed plugin-to-host recording command + contract. RAW and PDQ receipts are correlated by immutable run ID and actual + finalized paths; the workflow never claims filesystem atomicity. +7. Raw photodiode arrays do not cross JSON. A1 consumes small live summaries and + parses finalized PDQ data for replayable scientific results. + +## Safety and synchronization + +- Only the canonical live-worker instances may effect hardware. GUI mirrors, + replay, and offline instances are fail-closed. +- Duplicate request IDs return the original terminal response without repeating + an effect. +- Lease expiry, replay transition, plugin disable, worker shutdown, or hard fault + revokes control and requests output-off/finalization. +- Current PDA1 frames do not carry a shared modulation/configuration revision. + Ordered ACKs establish operational order, but scientific cross-port identity is + reported as `UNSYNCED` until firmware supplies a common epoch or marker. + +## Consequences + +- A1/A2/A3 can reuse the same owner services without duplicating serial code. +- The manual plugins remain independently useful and testable. +- ADR 005's statement that each experiment plugin owns the serial port no longer + applies to A1/A2/A3; exclusive ownership now belongs to the two device plugins. +- ADR 006's two-port/two-owner split becomes the stable architecture instead of a + temporary commissioning simplification. + diff --git a/docs/adr/008-stage-a-optical-waveform-inversion.md b/docs/adr/008-stage-a-optical-waveform-inversion.md new file mode 100644 index 0000000..fc599a7 --- /dev/null +++ b/docs/adr/008-stage-a-optical-waveform-inversion.md @@ -0,0 +1,67 @@ +# ADR 008 — Optical waveform inversion for the Stage-A modulator + +- **Status:** Accepted +- **Date:** 2026-07-20 +- **Relates to:** ADR 006 (two-plugin split), `stage-a-controller` waveform drive + +## Context + +The Pockels/PBS amplitude modulator has a `sin²` voltage→transmission transfer. +A pure DAC sine (`DAC_SINE`) therefore produces a distorted optical waveform, +and a 50 % bias only approximately linearises the small signal. The A1 +measurement wants a clean optical target — ideally a **log-intensity** sine, +because the event camera responds to changes in `ln I`. + +Producing that target requires driving the DAC with the *inverse* of the `sin²` +lobe, `V(u) = V_null + (2Vπ/π)·arcsin√u`, which is not a sinusoid. The existing +firmware only synthesises a pure sine from a fixed 256-entry table scaled between +`min`/`level`, so it cannot emit the warped shape as-is. The command line is also +capped at 192 bytes, too small to upload a 256-code table inline. + +## Decision + +1. **Own the inversion in the modulation plugin.** `waveform.rs` computes a + 256-entry DAC warp table from an `OpticalTarget` (`LogSine`/`LinearSine`), the + requested depth `a`, and a `LobeInversion { v_null_dac, v_pi_dac }`. It refuses + (never clamps) a drive whose codes leave `0..4095`. +2. **Keep the inversion parameters settable.** `V_null` and `Vπ` are entered in + DAC codes; no measurement rig is required to start. The scientifically clean + **measured LUT** (sweep constant codes, log the photodiode, freeze the table) + is a documented follow-up that drops in behind the same `warp_table` interface. +3. **Send parameters, not the table, over the wire.** The compact + `MOD wave=WARP freq_mhz=… target=… a_milli=… v_null=… v_pi=…` command fits the + 192-byte limit; the firmware rebuilds the identical table with the same formula + (`stimulus_mod::normalisedIntensity` + `dacForU`) and plays it back through a + `warpIsr`. A chunked table-upload command is the future path for the measured + LUT, which cannot be parameterised. +4. **Preserve `DAC_SINE`.** The pure DAC sine (firmware `SINE`) is unchanged and + remains the default for non-optical work. +5. **Separate drive from measurement.** The requested `a` is only a drive target. + The realised optical depth is always the photodiode-measured `a` + (rejected-complement corrected in the `stage-a-io` estimator), never the + commanded value. +6. **Keep drive method orthogonal to waveform mode (2026-07-23 amendment).** + `MANUAL` defines a DAC band from Power + Min threshold; `CALIBRATED` derives + one from `V_null`, `Vπ`, `I_k`, and `a`. All five waveform modes remain + available with both methods. Manual optical modes pass their DAC endpoints + through the forward `sin²` transfer to derive `(I_k, a)`, then reuse the same + inversion path. The separate `max_level` setting is the hard ceiling for + every drive; it is no longer merely the upper bound of the Power slider. +7. **Treat constant hold separately from modulation headroom (2026-07-23 + amendment).** `CONST` maps `I_k` directly through the inverse lobe and ignores + `a`; periodic modes retain the `I_k·exp(a/2) ≤ 1` ceiling. A rejected + calibrated setting is rolled back so displayed settings always describe the + command that can actually be sent. + +## Consequences + +- The plugin, the `stage-a-io` mock, and the firmware share one small parameter + contract and one formula; the inversion math is duplicated in Rust and C++ but + covered by the Rust round-trip tests (`sin²(warp) ≈ target`). +- Method changes only the operating-band source; mode remains a pure waveform + choice. The UI can therefore hide inactive parameters without filtering modes. +- Real optical output on hardware depends on firmware that supports the `WARP` + command; until flashed, the mode is exercisable only against the in-process + mock and the unit tests. +- The measured-LUT upgrade and the eventual `EXT_TRIGGER` camera marker (see the + A1 analysis brief) remain the two open scientific accuracy items. diff --git a/docs/adr/009-stage-a-a1-recording-coordinator.md b/docs/adr/009-stage-a-a1-recording-coordinator.md new file mode 100644 index 0000000..c56c977 --- /dev/null +++ b/docs/adr/009-stage-a-a1-recording-coordinator.md @@ -0,0 +1,107 @@ +# ADR 009 — Stage-A A1 as a focused recording coordinator + +- **Status:** Accepted +- **Date:** 2026-07-23 +- **Relates to:** ADR 005 (device ownership), ADR 006 (two-plugin split), + ADR 007 (owner orchestration — the earlier, broader orchestrator), + [Stage-A A1 Analysis](../features/stage-a-a1.md), + [Stage-A A1 Automation](../features/stage-a-a1-automation.md) + +## Context + +The A1 measurement records, for one illumination `I_k` and frequency `f`, several +runs while sweeping the modulation depth `a`. Each run must persist the camera +**RAW** stream, the photodiode **PDQ** stream, and enough configuration to +reproduce and analyse it offline — named consistently so repeats of an `(I_k, f)` +pair stay grouped. + +The previous A1 plugin (ADR 007, then the live-analysis MVP that superseded it) +was a *read-only* surface: it folded events into quicklooks and offered a manual +response-curve, but **recorded nothing**. Operators had to start/stop the camera +and photodiode recordings separately, with no shared naming and no single place +capturing the modulation settings and measured `a`. Its controls had also drifted +away from the real workflow: a "Capture camera events" toggle that recorded +nothing, an obsolete fallback frequency and phase-bin width, and interim +phase-anchoring knobs (event latency, self-align) that the now-reliable +`EXT_TRIGGER` makes unnecessary. + +## Decision + +1. **A1 becomes a focused recording coordinator.** One *Start recording* button, + a chosen **folder**, a per-`(I_k, f)` **measurement id** (auto-default, + regenerate, or edit), and a **duration** drive a small ordered state machine: + start and acknowledge the host camera recorder; connect and lease the + photodiode; open and acknowledge the PDQ; run for the requested duration; + atomically finalize the PDQ and release its lease; stop and acknowledge the + camera; then write an A1 config sidecar. This order keeps PDQ cleanup inside + the live-effects window and starts the timer only after both streams exist. + It deliberately **re-adds** recording orchestration that the + live-analysis MVP had dropped — in a narrow form: only camera + photodiode + recording, no drive/lease of the modulation device. + +2. **A1 never drives the Teensy.** The optical drive is armed in the modulation + plugin. A1 only *reads* the published `ModulationStateV1` snapshot into the + sidecar. Reintroducing the modulation drive (settle detection, amplitude + sweep) stays on the [automation roadmap](../features/stage-a-a1-automation.md). + +3. **Consistent naming, recorder-owned directories.** Files share an + `_` stem under an `/` subfolder. The camera RAW path is + relative to the **host output root** and the PDQ path relative to the + **photodiode data root** — each recorder confines its own writes, so A1 cannot + force a single absolute directory. The A1 config sidecar is written under + `//` and records the *resolved* paths of both files, so the + set is linked regardless; pointing all roots at the same experiment directory + co-locates everything physically. + +4. **Camera biases stay owned by the host recorder.** The host writes a companion + `.toml` next to the RAW containing the camera config (biases, ROI). A1 + cannot read biases itself; its sidecar cross-references that file and also + passes the key parameters as recording metadata, which the host and photodiode + embed in their own sidecars. + +5. **Two live quicklooks, clearly scoped.** Keep the **rolling half-period + response** `S_p(t)` (live sanity: are events appearing, is ON/OFF timing sane?) + and the **response probability** `q_p` (binary pixel-cycle statistic vs measured + `a`). Drop the phase-bin rate plot. The authoritative `q_p(a, f)` fit is an + **offline** computation over the recordings; the live `q_p` is a quicklook. + + **`q_p` windows: auto by default, pilot-frozen per row.** Because the + `EXT_TRIGGER` fixes the phase, ON and OFF fall in opposite half-cycles, so the + windows are found directly from the current fold — each anchored on its + histogram peak and grown outward until it drops below a floor (default 10 % of + the peak) or the opposite polarity dominates. This replaces the old + manual-pilot *button* and its window-threshold / self-align knobs. + + The window phase depends on the event latency, which is a *phase* shift `τ·f` + (negligible at low `f`, up to a full cycle at high `f`) and drifts with `I_k`, + so windows must be fixed **per `(I_k, f)` row** and held across the `a`-sweep. + A **Record pilot** action therefore freezes the auto-windows for the row and + writes them into the pilot recording's sidecar; **Record background** captures + the floor `q0`. Both are keyed to the measurement id (one id = one row) and are + auto-reloaded by scanning the measurement folder, so returning to a row reuses + its frozen windows. The live `q_p` remains a quicklook — the authoritative fit + still freezes windows offline from the brightest run. + +6. **Lean the trigger surface.** With `EXT_TRIGGER` now reliable, remove the + fallback frequency, the phase-bin width, the event-latency shift, and the + response-curve self-align/threshold knobs. The trigger marker spacing *defines* + `T`; the modulation acknowledged waveform is the only fallback. + +## Consequences + +- A1 now declares `host_commands = ["start_recording", "stop_recording"]` in its + manifest and holds a photodiode lease while recording (the photodiode's manual + recording UI is locked during that window). The first host-command use triggers + a one-time GUI consent prompt. +- A1 writes one file itself (the `.toml` sidecar) via `std::fs` — a small, bounded + write, not a PDQ/serial writer; hardware ownership is unchanged. +- A recording reports success only after complete host finalization and a valid + typed PDQ finalization receipt. The UI keeps one concise phase/result message, + not a rolling internal log. +- The host returns to Preview before delivering its final receipt, which keeps + repeated recordings and automated sweeps live without an extra operator step. +- True single-directory co-location is a **configuration** convention (align the + recorder roots), not something A1 enforces. Enforcing it would require host and + photodiode path changes and is out of scope. +- The contract and ABI are unchanged: every message used already exists + (`HostCommand`, `PhotodiodeCommandV1` lease/begin/finalize). diff --git a/docs/adr/010-stage-a-a1-amplitude-sweep.md b/docs/adr/010-stage-a-a1-amplitude-sweep.md new file mode 100644 index 0000000..ab036fb --- /dev/null +++ b/docs/adr/010-stage-a-a1-amplitude-sweep.md @@ -0,0 +1,72 @@ +# ADR 010 — Stage-A A1 amplitude sweep via leased optical-depth retargeting + +- **Status:** accepted (2026-07-23) +- **Relates to:** ADR 007 (owner orchestration), ADR 009 (recording + coordinator), [Stage-A A1 Automation](../features/stage-a-a1-automation.md) + +## Context + +The A1 workflow records a response curve `q_p(a, f)`: several recordings at +different modulation depths `a` for one `(I_k, f)` row. With the manual +coordinator (ADR 009) the operator had to retarget the drive in the modulation +plugin and press *Start recording* once per amplitude. The automation roadmap +(§1–§4 of the automation brief) calls for a scoped control path: sweep only +`a`, never the rest of the drive. + +Two structural gaps blocked this: + +1. **No semantic "set depth" command.** The modulation service only exposed + `SetWaveform` (raw DAC band) and `PrepareA1`. Sweeping `a` through raw DAC + values would duplicate the optical-inversion math (ADR 008) and the + calibration state (`V_null`, `Vπ`, `u_k`) outside their owner. +2. **Momentary buttons never reached the live worker.** The host runs a UI + mirror and a live worker per plugin; button presses land on the mirror via + `set_setting(key, true)`, while the worker only receives the settings + snapshot built from `get_setting`. Buttons that returned `false` lost every + press (the root cause of the dead record buttons). + +## Decision + +**1. `ModulationCommandV1::SetOpticalDepth { depth_a_milli }`** (contract +addition, additive to V1). Under an automation lease the modulation owner +re-derives its armed drive with the new depth through the same +`drive_command()` builder the operator path uses; everything else (waveform +shape, frequency, `u_k`, calibration, power cap) stays as armed. The owner +rejects the command when no device link is open, when the armed drive cannot +express a depth (manual DAC method, constant mode), or when the derived drive +violates its own safety validation. The command is applied immediately +(`Applied`), not revision-tracked: the sweep's ground truth for "the drive is +really there" is the photodiode-measured `a`, not a firmware ACK. + +**2. The sweep lives in A1** as a small state machine layered *on top of* the +ADR 009 coordinator: `AcquiringLease → (per point) SettingDepth → Settling → +Recording → …release`. Per point it renews the modulation lease, retargets the +depth, waits until the photodiode-measured `a` holds the target tolerance +(±10 %, at least ±0.05) for the configured dwell (30 s cap, then it records +anyway — the sidecar stores the measured `a`), and hands off to the unchanged +recording coordinator (`…_pNN` stem tag, `sweep.requested_a` / `point_index` / +`point_total` in the sidecar). Any rejection, timeout, or failed point aborts +the sweep and releases the lease (`safe_off = false` — the drive holds; safety +remains the owner's lease-expiry job). + +**3. Press counters for momentary buttons.** Every A1 button exports a +monotonic press counter from `get_setting`; `set_setting` interprets `true` as +a local click and a counter advance as one forwarded press edge, adopting the +first-seen value silently (reloads must not replay presses). The plugin-API +`Button` doc now records this idiom, and `SettingKind::Button` gained an +`enabled` flag (serde-default `true`, backward compatible in both directions) +so prerequisite-less presses can be prevented in the UI instead of rejected +after the fact. + +## Consequences + +- A1 now drives exactly one modulation parameter, under a lease, through the + contract — the "A1 owns no hardware" boundary narrows to "A1 may retarget + the armed drive's depth while leased" (the focused re-introduction ADR 007 + anticipated). +- Manual modulation settings stay locked during a sweep (lease lock), and the + operator's own `depth a` re-applies on the next modulation settings sync + after release. +- The press-counter idiom is the sanctioned pattern for momentary controls in + dual-instance plugins; requested-state booleans (`connect`, `record`, + `protocol_run`) remain correct as-is. diff --git a/docs/adr/011-stage-a-pockels-transfer-calibration.md b/docs/adr/011-stage-a-pockels-transfer-calibration.md new file mode 100644 index 0000000..30070f1 --- /dev/null +++ b/docs/adr/011-stage-a-pockels-transfer-calibration.md @@ -0,0 +1,151 @@ +# ADR 011 — Measured Pockels transfer calibration in the modulation plugin + +- **Status:** Accepted +- **Date:** 2026-07-25 +- **Relates to:** ADR 006 (two-plugin split), ADR 008 (optical waveform + inversion), ADR 010 (amplitude sweep / press-counter idiom), + [Stage-A Pockels Transfer Calibration](../features/stage-a-pockels-calibration.md) + +## Context + +ADR 008 made `V_null`/`Vπ` settable and named the **measured LUT** as the +follow-up. In practice they stayed two bare number fields whose tooltip told the +operator to measure them while the software offered no way to do so. Nothing +related a DAC code to an observed photodiode value, so the whole calibrated +drive rested on numbers typed in from a nominal datasheet — exactly what the +knowledge base warns against (`methodology/pockels-waveform-linearisation.md` +§1: "Do not use nominal `Vπ` as the measurement calibration"). + +## Decision + +### 1. The modulation plugin owns the calibration + +It already owns `V_null`/`Vπ` and the DAC. It reads photodiode levels **read +only** from the control-snapshot broadcast (the same bus A1 reads for the +measured `a`), so no lease, no service command, no coordinating plugin, and no +PDQ recording are involved. The alternative — a lease-based cross-plugin +protocol like ADR 010's sweep — would have moved the calibration state away from +the parameters it calibrates for no gain. + +### 2. `PhotodiodeStreamV1.level` — one additive V1 field + +`PhotodiodeLevelV1 { mean_volts, peak_to_peak_volts, sample_count, +end_sample_index, clipped }`, `#[serde(default)]`. + +`PhotodiodeOpticalSummaryV1` could not serve: it reports *contrast* not level, +applies the geometry transform (which needs an anchor this reading must not +depend on), and **refuses** on clipping or missing headroom — precisely at +`V_null`, where the reject-port detector is brightest. The level is deliberately +fail-open where the optical summary is fail-closed, and always **raw** detector +volts, never the plugin's RAW/EXCITATION display transform. + +`end_sample_index` makes settling *provable*: a point is accepted only from a +window that began after its code was commanded plus a settle margin, on the +device sample clock. No shared wall clock, no sleeps, immune to tick jitter. + +### 3. The detector geometry is an input, not an inference + +The initial design assumed a free-signed amplitude would let the fit *identify* +the port. It cannot. Since `sin²` is symmetric about its peak, +`(v, p₀, p₁)` and `(v + Vπ, p₀ + p₁, −p₁)` describe the measured curve +*identically* — the data cannot say which extremum is zero excitation. This is a +fact about the optics, so it is asked (`Detector port`, default `REJECT PORT`, +which `setup/optical-path.md` settles by construction) and the fit selects the +matching representation. Guessing would place `V_null` a quarter wave off and +silently run the drive on the inverted branch. + +### 4. One-dimensional harmonic fit, not a nonlinear solve + +`sin²(x) = (1 − cos 2x)/2` makes the model a constant plus one sinusoid of +period `2Vπ`, which is linear in its quadrature components. For each candidate +`Vπ`, the phase (hence `V_null`) and both amplitudes come from a 3×3 solve, so +only `Vπ` is searched — a log-spaced scan plus a golden-section refine. + +The rejected alternative, seeding the period from the measured extrema, breaks +on the sweeps that matter: at a realistic `Vπ ≈ 860` the DAC range holds ~2.4 +lobes and the global extrema can sit whole periods apart. + +Where several nulls are valid, the **lowest** in-range one wins: least voltage +across the crystal, most headroom, and predictable for the operator. + +### 5. `enabled` is computed from mirrored settings only + +`settings_schema()` is rendered by the **UI mirror**, which by construction +never owns the device link, a lease, a running sweep, or a fit — all of that +lives on the live worker. A first cut gated the calibration buttons on +`calibration_blocker()` and `fit.is_some()`, which disabled them *permanently*: +the mirror can never satisfy either. The buttons now gate on the one +prerequisite the mirror does know (the operator asked to connect), and the +authoritative interlocks stay worker-side, reported through the status entries +the host already takes from the worker. + +**Rule for this repo:** a `SettingKind::Button { enabled }` may only depend on +state that is itself a setting. Anything else is invisible to the instance that +renders it. The same trap bit the press counters — a baseline folded into the +counter made a fresh worker swallow the operator's first press, so the modulation +plugin now uses A1's `PressLatch` (separate `counter` and `seen`) verbatim. + +### 6. The sweep owns the DAC, so `send_modulation` is silent while it runs + +`apply_live_plugin_snapshot` writes **every** settings key to the worker on +every sync, and most of this plugin's drive handlers call `send_modulation()` +unconditionally rather than on change. Each sync therefore re-armed the +operator's waveform on top of the code the sweep had just commanded: the board +spent the sweep playing the armed drive, every point measured the same +waveform-averaged level, and the fit correctly reported `NoModulation` on a +bench where the light was plainly modulating. + +`send_modulation` now returns early while a sweep is in flight, the same shape +as the existing automation-lease guard — a sweep is simply another owner of the +DAC. Settings changed mid-sweep are withheld rather than rejected, and land on +the board when the sweep finishes: the restore prefers the *current* drive and +falls back to the command captured at sweep start. + +### 7. Robust refit, and fit quality warns rather than blocks + +The first cut refused to apply a fit whose residual exceeded 2 % of the detector +span. On the bench that gate fired at 20.8 % on a sweep whose plot looked +correct, and withheld a usable calibration. + +Measuring the failure modes on a realistic small-signal sweep settled it: 5 mV +of noise gives 3.1 %, drift 3.2 %, hysteresis 5.5 % — but a **single stray +point gives 9.9 % while leaving `Vπ` accurate to three codes**. Residual and +correctness are not the same axis, so a residual threshold is the wrong thing to +block on. (A genuinely wrong fit — an amplifier compressing the top of the +range — gives 15.2 % *and* a `Vπ` off by 250 codes, which the plot shows +plainly.) + +Two changes follow. The fit now runs twice, dropping points beyond `6 × median` +absolute residual before refitting — a median cut, because mean and standard +deviation are themselves inflated by the points being sought. And every quality +measure became a warning; the only meaningless case, no full lobe inside the +commandable range, is already refused inside `fit_transfer`, so the separate +coverage gate was dead code and was removed rather than kept. + +Applying still re-validates the resulting drive and rolls back if it cannot be +armed. The sweep restores the pre-sweep drive on every exit path, and refuses to +run while a lease or protocol owns the DAC. + +### 8. `ModulationStateV1.calibration_id` — one additive V1 field + +Set when a measured fit is applied, `None` when the lobe was typed in by hand, +so a consumer's sidecar can cite which inversion produced a run's optical depth. +Previously unrecoverable. + +## Consequences + +- The reported detector level at the null is a **lower bound** on the + total-power anchor `I_tot`, not the anchor: on the reject port the residual + transmitted floor is not separable from it (knowledge base §4.4). The plugin + labels it as such and derives no maximum achievable `a` from it. Freezing a + real anchor still needs a transmitted-port power measurement. +- `V_null`/`Vπ` need neither a dark measurement nor an anchor, because the + fitted offset and amplitude absorb both. That is what keeps this one button + instead of a protocol. +- Ascending and descending passes are both recorded, so the hysteresis figure + the knowledge base's acceptance test 1 asks for comes out of the normal run. +- Still an analytic `sin²` inversion, not a measured LUT. The archived record + stores the points a LUT would need, so ADR 008's follow-up remains open behind + the same `warp_table` interface. +- A static calibration must never be used to correct dynamic roll-off; doing so + would manufacture the Bode curve A1 exists to measure. diff --git a/docs/architecture.md b/docs/architecture.md index c67c61f..dcba238 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,6 +71,28 @@ Key properties: New plugins should prefer this shared host contract over duplicating pixel scale or sensor geometry in plugin-local defaults. +## Frame-Independent Plugin Services + +Hardware workflows use the host-routed control plane rather than the per-frame +JSON context. One canonical live-worker instance owns effects; GUI mirrors, +replay, and offline instances remain fail-closed. Requests target stable manifest +IDs and carry semantic operation names, request IDs, leases, and explicit +responses. Device plugins validate and execute their own operations; the host is +only the router. + +Stage-A uses a serde-only companion contract so `stage-a-a1` can orchestrate +`stage-a-modulation` and `stage-a-photodiode` without linking their implementation +crates or opening their serial ports. See +[`docs/adr/007-stage-a-owner-orchestration.md`](./adr/007-stage-a-owner-orchestration.md). + +Not every cross-plugin dependency needs that machinery. Because the host +broadcasts each plugin's `control_snapshots()` to every plugin's inbox, a plugin +that only needs to *read* another's published state can do so directly — no +lease, no service request, no router round-trip. The Pockels transfer +calibration reads photodiode levels this way while driving only its own DAC: +[`docs/adr/011-stage-a-pockels-transfer-calibration.md`](./adr/011-stage-a-pockels-transfer-calibration.md). +Reserve the leased service path for *commanding* hardware someone else owns. + ## Host Views Plugins declare host-rendered datasets and views through: diff --git a/docs/features/README.md b/docs/features/README.md index 16ed650..2ff4830 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,8 +5,12 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. -- [Stage-A Modulation](./stage-a-modulation.md) — capped power slider + constant/sine/square laser-modulation drive on the command port, applied immediately. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`. +- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. +- [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from settable `V_null`/`Vπ`. +- [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`Vπ` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`, plus the geometry-corrected optical depth `a`. +- [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. +- [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-automation.md b/docs/features/stage-a-a1-automation.md new file mode 100644 index 0000000..529a10d --- /dev/null +++ b/docs/features/stage-a-a1-automation.md @@ -0,0 +1,110 @@ +# Stage-A A1 Automation — Plan (partially implemented) + +- **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) +- **Status:** **Partially built.** §1 (scoped A1→modulation control path), §2 + (settle detection), §3 (per-point recording) and the single-row core of §4 + (the amplitude loop) now exist as the **Start sweep** button — see + [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md). Still open: scout phase, + randomized point order, multi-`f`/multi-`I_k` rows, `UNIDENTIFIABLE` stop + rule, and the offline `a50` fit (§5–§6). +- **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), + [Optical Waveform Drive](./stage-a-optical-waveform.md), + [ADR 007](../adr/007-stage-a-owner-orchestration.md) (the earlier full + orchestrator this deliberately re-adds in a *focused* form), + [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) (the per-recording + RAW + PDQ + sidecar coordinator, now built — see §3). + +> **Update (2026-07-23):** the manual per-recording coordinator in §3 now exists +> (ADR 009): one *Start recording* button records camera RAW + photodiode PDQ + +> an A1 config sidecar per `(I_k, f)` measurement, over an operator-set duration. +> +> **Update (2026-07-23, later):** the single-row amplitude sweep now exists +> (ADR 010): *Start sweep* leases the modulation owner, retargets the armed +> drive per point via `SetOpticalDepth`, waits for the photodiode-measured `a` +> to settle (tolerance + dwell, 30 s cap), and records each point through the +> §3 coordinator with `sweep.requested_a` / `point_index` / `point_total` in +> the sidecar. Remaining below: scout/randomized order, multi-`f`/`I_k` +> iteration, the `UNIDENTIFIABLE` rule, and the `a50` fit. + +## Goal + +Semi-automate the researcher's normal A1 workflow: for one illumination `I_k` +and frequency `f`, sweep the modulation depth `a` and record the response curve +`q̂_p(a,f)`, automatically starting/stopping/saving each recording with proper +naming and full parameters. Later, repeat over `f` and over `I_k`. + +## Already in place (the foundation) + +- Two live plots `r_{p,k}` and `S_p`, plus the response curve `q̂_p(a)`. +- Marker-anchored phase folding from the firmware **phase-0 EXT_TRIGGER**; the + trigger **defines the frequency** (measured marker spacing); event-latency + handling (marker shift + optional self-alignment). +- **Pilot capture** → frozen ON/OFF phase windows; manual "record point" + appends `(measured a, q_on, q_off)` to the curve. +- **ROI + masked pixels** come from the host camera config (`GlobalSettings`); + `N_valid = |ROI| − |masked|`. +- Photodiode-measured **`a`** (rejected-complement geometry) published and + surfaced in A1. +- Optical drive with a **fixed operating point `I_k`** and swept `a` + (`OPTICAL_LOG_SINE`/`OPTICAL_LINEAR_SINE`), power-capped. +- Events sourced exactly from the retained **EventStore** over a sliding window. + +## To build (the automation) + +### 1. A1 → modulation control path (scoped) +Re-introduce a *focused* control path (the contract + modulation plugin still +support it): acquire a modulation lease, set the optical drive +`(target, I_k, a, f, V_null, Vπ)`, start, stop, release. No full workflow zoo — +just set-amplitude / start / stop. Fixed `I_k`, only `a` varies within a curve. + +### 2. Settle detection +Before collecting a point, wait until the photodiode confirms the optical +waveform has stabilised at the new `a` — e.g. the published `measured_a` is +within tolerance of the target and clip-free for a short dwell. Only then start +the counting window. + +### 3. Per-point recording (proper naming + parameters) +For the pilot, background, and every amplitude point, orchestrate: +- host camera **RAW** recording (re-add the host recording commands), +- photodiode **PDQ** recording (`stage-a-photodiode` begin/finalize), +- a **config sidecar** with everything needed to reproduce/replay: `I_k`, `f`, + requested + measured `a`, ON/OFF windows, ROI, masked pixels, `N_valid`, `M`, + latency, biases, run/session ids, timestamps, settle/clip status. +- **Deterministic naming**: `/A1////-...`. + +### 4. Sweep state machine +`SAFE → BACKGROUND(a=0) → PILOT(high, freeze windows) → SCOUT(locate the +transition) → SWEEP(5–7 settled amplitudes spanning ~10–90 %, randomized or +alternating order) → NEXT_FREQUENCY → … → NEXT_ILLUMINATION`. Windows are frozen +from the pilot and **must not** be re-derived from measurement points. + +### 5. Classification and stop rules +- `q̂_p(a,f) = (1/(N_valid·M)) Σ_i Σ_c z_{i,c,p}`, ON/OFF independent + (already implemented for a single point — the sweep just repeats it). +- If the transition cannot reach ~90 % within the safe amplitude range, mark the + frequency **`UNIDENTIFIABLE`** (do not keep increasing `a`). + +### 6. Final fit (per curve) +Fit the background-floor logistic `p(a) = p0 + (1−p0)·logistic((a−a50)/slope)` +to get **`a50`** with a cycle/spatial-tile bootstrap interval; label quality +(VALIDATED / DEGRADED-no-background / UNVALIDATED-no-pilot). (This was the old +`response.rs`; re-add as the sweep's summary output.) + +## Known hard problem (only if per-cycle cross-stream correlation is ever needed) + +The camera trigger and the photodiode stream marker are the *same* firmware +phase-0 on two clocks, consumed **independently** today — nothing pairs cycle +*k* across the two streams, and nothing needs to. If a future metric correlates +per-cycle optical depth with per-cycle camera response, ordinal matching is +fragile (start offset, asymmetric drops, drift). The robust fix is a **cycle +counter** in the PD `MarkerPayload` plus a **distinctive fiducial pattern** +(e.g. a periodic marker cycle) visible in both streams to align on and detect +drops — see the controller's `a1-marker-cycles.md`. + +## Open decisions to confirm at build time + +- Amplitude list: explicit list vs. min/max/count range (randomized order). +- Mask source already resolved: host `GlobalSettings.masked_pixels`. +- Whether RAW+PDQ per point is always on or gated by a "record" toggle + (user already asked for full RAW + PDQ + params per recording). +- Live is a quicklook; the **RAW/PDQ replay is authoritative** for the final fit. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md new file mode 100644 index 0000000..0b72be2 --- /dev/null +++ b/docs/features/stage-a-a1.md @@ -0,0 +1,204 @@ +# Stage-A A1 Analysis + +- **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) +- **Status:** Recording coordinator + live quicklooks + amplitude sweep +- **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), + [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button + press forwarding) +- **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) + +## Purpose + +A1 has two jobs on the Stage-A bench, both deliberately thin: + +1. **Recording coordinator.** One *Start recording* button records the camera + **RAW** stream and the photodiode **PDQ** stream together for a fixed duration, + grouped under a per-`(I_k, f)` measurement id, and writes an A1 **config + sidecar** (`.toml`) linking them with everything needed to reproduce and + analyse the run offline. +2. **Live sanity quicklooks.** The rolling half-period response `S_p(t)` and the + response probability `q_p`, folded on the modulation period `T`. + +A1 owns no hardware and never drives the Teensy. The optical drive is armed in the +modulation plugin; A1 only *reads* its published settings. + +## The recording workflow + +The experiment sweeps the modulation depth `a = ln(I_max/I_min)` at a fixed +illumination `I_k` and frequency `f`, taking several recordings per `(I_k, f)` +pair (a background `a≈0`, a bright pilot, then settled amplitudes). One +**measurement id = one `(I_k, f)` row**; every recording under it lands in the same +folder. A1 makes each recording one button press: + +| Control | Meaning | +|---|---| +| Output folder | where the A1 config sidecar is written (recommended shared experiment root) | +| Measurement id | one per `(I_k, f)` row; auto-generated default, editable, or press **New id** | +| Sweep min a / max a | the `a`-range for this row; the **Start sweep** button records it, and it is stored in every sidecar | +| Sweep points (count) | how many amplitudes Start sweep records, spaced evenly over `[min a, max a]` | +| Sweep settle (s) | dwell the photodiode-measured `a` must hold the target (±10 %, ≥±0.05) before each sweep recording; 30 s cap, then it records anyway | +| Duration (s) | each recording auto-stops and finalizes after this | +| Start recording (sweep point) | start camera RAW → connect and lease photodiode → start PDQ → auto-stop and save both → sidecar | +| Start sweep (record all points) | per point: lease the modulation owner → retarget the calibrated drive to `a_i` → settle → one recording (`…_pNN`) → next point | +| Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | +| Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | +| Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | + +The record and sweep buttons stay **disabled until an output folder is +selected**. + +For manual recordings A1 never drives the Teensy: set the drive (high `a` for +the pilot, `a≈0` for the background) in the modulation plugin, then press the +matching button — the recording captures whatever `a` is currently set. + +**The sweep is the one scoped exception.** Start sweep leases the modulation +owner (`SERVICE_STAGE_A_MODULATION_CONTROL_V1`) and, per point, issues +`ModulationCommandV1::SetOpticalDepth` — which only retargets the *depth* of the +drive the operator already armed (waveform, frequency, operating point `I_k`, +and calibration stay untouched; the owner refuses when a manual-DAC or constant +drive is armed). It renews the lease per point, waits for the +photodiode-measured `a` to settle, hands the point to the normal recording +coordinator, and releases the lease at the end or on abort. Sweep points +require `min a > 0` — record `a≈0` with the background button instead. Sidecars +of sweep recordings additionally carry `sweep.requested_a`, `sweep.point_index` +and `sweep.point_total`. After the sweep releases the lease, the drive holds the +last sweep amplitude until the operator's own `depth a` setting is re-applied +(any modulation settings change re-sends it). + +**Naming.** Files share an `_[_role]` stem under an `/` subfolder +(`_pilot` / `_background` tag the reference runs): + +- `/_.raw` — camera RAW, under the **host output root**, with the host's + own `.toml` sidecar (camera biases, ROI) written next to it. +- `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar, under the + **photodiode data root**. +- `/__config.toml` — the A1 sidecar, under the chosen output folder. + +Each recorder confines its writes to its own root, so A1 cannot force one absolute +directory (see ADR 009). Point the host output root and the photodiode data root +at the same experiment directory to co-locate everything; the A1 sidecar records +the *resolved* paths so the set stays linked either way. + +**A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize +timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the +acknowledged snapshot (frequency, center/amplitude DAC, waveform); the +photodiode-measured `a` (`measured_log_contrast`) and clip fractions; ROI + +masked-pixel count + `N_valid`; trigger info (marker-anchored, marker count, +measured period); and the resolved paths of the RAW (+ its camera-config sidecar) +and the PDQ (+ its sidecar). The **pilot** run additionally records the frozen +ON/OFF windows and the **background** run the floor `q0`, so returning to a +measurement (folder + id) auto-reloads them for the `q_p` plot. + +**Mechanism.** A small control-plane state machine in `process_control` starts +the host camera recorder first and waits for its receipt. Only after the host +has completed the Preview → Recording switch does A1 connect and lease the +photodiode and open the PDQ with the same run id. The duration begins when the +PDQ start receipt arrives, so setup time is never deducted from the requested +recording. On completion A1 atomically finalizes the PDQ and releases its lease +while camera effects are still live, then stops the host recorder, waits for its +final receipt, and writes the config sidecar. A recording is successful only +when the host receipt is complete and the photodiode returns a valid finalized +receipt with both PDQ paths. The status panel shows only the current phase and +one concise result or error message; it does not render an internal event log. +A1 declares `host_commands = ["start_recording", "stop_recording"]` in its +manifest. Every role uses this same lifecycle. + +**Host-side note.** The camera RAW leg restarts the host pipeline into +Recording mode and stops it again at finalize. After the file is finalized, the +host restores Preview before returning the receipt, so a sweep or another button +press can start the next recording automatically. + +**File locations** (three roots, point them at the same experiment directory): +`//.raw` (+ host `.toml`), +`//_pd.pdq` + `_pd.json`, and +`//_config.toml`. + +## The two live plots + +Both fold the camera event stream on `T` (from the firmware phase-0 `EXT_TRIGGER` +marker spacing, which *defines* the frequency; the modulation acknowledged waveform +is the only fallback). Enable **Live analysis** to keep them updating. + +Marker hygiene: preview windows overlap, so the same trigger edge arrives on +several consecutive frames — the marker buffer is sorted and deduplicated on +every merge (duplicates used to fail marker validation and blank the plots). +When marker validation still rejects a fold (dropped-trigger jitter), the +quicklook falls back to the free-running fold on `T` instead of going empty. + +1. **Rolling half-period response** + + ```math + S_p(t) = \frac{N_p(t-T/2,\,t]}{N_\text{valid}} + ``` + + events per valid pixel in the trailing half-cycle, ON and OFF. A live indicator: + are events appearing, does the ON/OFF timing look sane, is the response + saturating? It counts *every* event, so a noisy pixel weighs heavily — it is a + quicklook, not the response metric. + +2. **Response probability** `q_p` + + ```math + z_{i,c,p} = \mathbf{1}[\text{pixel } i \text{ fires in } W_p \text{ during cycle } c], + \qquad + \hat q_p(a,f) = \frac{1}{N_\text{valid} M}\sum_i\sum_c z_{i,c,p} + ``` + + the fraction of valid pixel-cycles that fire at least once in the ON/OFF phase + window `W_p` — each pixel-cycle counts **once** (unlike `S_p`). The windows come + from the row's **pilot** when one has been recorded (frozen, held across the + whole row), otherwise from the trigger-anchored fold automatically: since the + `EXT_TRIGGER` fixes the phase, ON and OFF live in opposite half-cycles, so each + window is anchored on its histogram peak and grown outward until events fall + below the **window floor** (default 10 % of the peak) or the opposite polarity + takes over. `Record point` appends one `(measured a, q_on, q_off)` dot. The ROI + and masked pixels come from the augur-rs camera config + (`N_valid = |ROI| − |masked|`). + + **Why the pilot is per row.** The window phase depends on the event latency, + which is a *phase* shift `τ·f` — negligible at low `f`, up to a full cycle at + high `f` — and also drifts with `I_k`. So the windows must be defined **per + `(I_k, f)` row** and held fixed across that row's `a`-sweep (re-deriving them + per amplitude would bias the curve). One pilot per measurement id captures that + exactly. This live `q_p` stays a quicklook; the **authoritative** `q_p(a, f)` + fit (`a50`, background floor) is computed offline from the recordings. + +## Button presses across the UI-mirror / live-worker split + +The host loads two instances of every dynamic plugin: a **UI mirror** (renders +the settings, never touches hardware) and the **live worker** (runs +`process_frame` / `process_control`, owns the recording state machine). A +`SettingKind::Button` click calls `set_setting(key, true)` **on the mirror +only**; the worker receives settings through the host's snapshot, which carries +whatever `get_setting` returns. A1 therefore exports every button as a +**monotonic press counter** (`PressLatch`): the mirror increments it per click, +the snapshot transports it, and the worker treats a counter advance as exactly +one press edge (the first value a freshly loaded worker sees is adopted +silently, so reloads never replay old presses). This is why the record buttons +used to do nothing — the presses died on the mirror. + +Related: A1 overrides `on_discontinuity` to ignore `SettingsChanged` (raised on +*every* settings sync of any plugin), so the response curve, pilot windows and +background floor survive ordinary UI interaction; source changes and seeks +still reset everything. + +## Where the inputs come from + +| Input | Source | +|---|---| +| camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()` | +| phase-0 markers | rising `frame.external_triggers()` — the host **banks trigger edges from dropped preview frames** into the next processed frame (drain-to-newest and the preview throttle drop whole frames; at low modulation frequencies the survivors alone rarely held 2 markers inside the analysis window) | +| modulation period `T` | measured from the `EXT_TRIGGER` marker spacing; else the modulation plugin's acknowledged waveform — which, since the board-echo fallback, includes the **operator-armed UI drive**, not only service-path (leased) targets | +| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) | +| ROI, masked pixels | augur-rs camera config (`CTX_GLOBAL_SETTINGS`) | + +## Tests + +`cargo test -p augur-plugin-stage-a-a1` covers trigger-defined period, marker-anchored +folding, ON/OFF separation of the rolling dataset, auto-window detection and the `q_p` +path, file-safe id generation, UTC timestamp formatting, the config-sidecar builder, +the pilot-window round-trip through the measurement folder, press-latch edge/baseline +semantics, the jittery-marker free-running fallback, sweep-point spacing, the +sweep-point sidecar fields, the ordered camera → PDQ → PDQ finalize → camera +finalize lifecycle (including envelope identity/revision and save location), and +the selective discontinuity reset. diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index 49279fa..e9077f9 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -7,10 +7,32 @@ ## What it is -The simplest possible laser-modulation control for the Stage-A bench: one power slider in DAC -codes (J23 output, `DAC1.4`), a mode select (`CONST`/`SINE`/`SQUARE`) with frequency -(0.01–2000 Hz) and a min threshold for the periodic modes, and a user-set **max limit** that caps -the slider so a device with a lower tolerated input voltage can never be overdriven from the UI. +Laser-modulation control for the Stage-A bench with two orthogonal axes: + +- **Drive method** defines the DAC operating band. `MANUAL` uses Power + Min threshold; + `CALIBRATED` derives it from `V_null`, `Vπ`, `I_k`, and optical depth `a`. +- **Mode** defines the shape that fills the band: `CONST`, `DAC_SINE`, `SQUARE`, + `OPTICAL_LOG_SINE`, or `OPTICAL_LINEAR_SINE`. All five remain available under both methods. + +The always-visible **max limit** is the hard DAC ceiling for every manual and calibrated drive. +The settings schema shows only the selected method's parameter block and refreshes when Method +changes; Manual is the default. + +| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +|---|---|---| +| `CONST` | hold `power` | hold the DAC code for `I_k` | +| `DAC_SINE` | DAC sine across the band | DAC sine across the band | +| `SQUARE` | DAC square across the band | DAC square across the band | +| `OPTICAL_LOG_SINE` | intensity log-sine across the band | intensity log-sine about `I_k` | +| `OPTICAL_LINEAR_SINE` | intensity linear-sine across the band | intensity linear-sine about `I_k` | + +Manual optical modes reuse the persisted `V_null`/`Vπ` lobe parameters and derive effective +`(I_k, a)` from the manual DAC band through the forward `sin²` transfer. Both optical modes then +use the same inversion path described in [Optical waveform drive](./stage-a-optical-waveform.md). + +`V_null`/`Vπ` are measured, not typed: the Calibration section sweeps settled `CONST` codes +against the photodiode and fits the lobe — see +[Pockels transfer calibration](./stage-a-pockels-calibration.md). Every accepted setting change is transferred to the Teensy **immediately** as one `MOD` command — no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the @@ -27,13 +49,35 @@ board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded val flow — bench control must work with no camera attached. `process_frame()` only disconnects defensively in replay mode. - Firmware output is **set-and-hold** (`stage-a-controller` ADR 002): disconnecting does not stop - the modulation. The power slider at 0 is the off switch. -- Safety invariants enforced plugin-side: `level ≤ max_level`, `min_level ≤ level`; the firmware - waveform peaks at `level` by construction. + the modulation. Manual Power at 0 drives 0 V; automation has an explicit `SafeOff` operation. +- Safety invariants enforced plugin-side: `min_level ≤ level ≤ max_level` for Manual and every + resolved calibrated/optical peak must be `≤ max_level`; invalid drives are refused. +- Status and commanded summaries include Method and the resolved `(lo, hi, hold)` DAC band. - `mock` port runs the firmware-faithful `MockController` in-process for hardware-free tests. +- The workflow-owner service and `WaveformV1` automation path remain exact-waveform contracts and + do not use the UI Drive method. +- **`SetOpticalDepth`** (ADR 010): under an automation lease the service can retarget the *depth* + `a` of the drive the operator armed — same `drive_command()` builder as the UI path, everything + else untouched. Refused with no device link, a manual-DAC method, or a constant mode; the derived + drive still passes all safety validation. Used by the A1 amplitude sweep. +- **Link watchdog**: the device thread exits after 5 consecutive serial failures (marking the + device disconnected/faulted), and the control tick reaps a finished device thread and + auto-reconnects with a 2 s backoff while `connect` stays requested. Previously a wedged or dead + link silently swallowed every queued command — the UI kept accepting mode changes while the + board held the old waveform. +- **`protocol_run` forwarding**: the UI mirror records the request and the settings snapshot + starts/stops the protocol on the live worker (which owns the device link); only value + *transitions* act, so re-applied snapshots cannot restart a finished protocol. +- **Board-echo `acknowledged` fallback**: the published `ModulationStateV1.acknowledged` now falls + back to a revision-0 target built from the board's `MOD`/`STATUS` echo (`mod_wave`, `mod_level`, + `mod_min`, `mod_freq_mhz`) when no service-path acknowledgement exists. UI-driven drives never + produce a service ACK, so consumers (A1's fallback modulation period) previously saw no waveform + at all for the normal operator workflow. WARP (optical) echoes map to `Periodic` — the fallback's + consumers only need the frequency. ## Verification -`cargo test -p augur-plugin-stage-a-modulation` — mock round trips: immediate transfer on slider -change, board-code echo, max-cap clamping (including schema regeneration), square drive with min -threshold, Output OFF. +`cargo test -p augur-plugin-stage-a-modulation` covers method/mode enum index round-trips, +conditional settings blocks, method-resolved bands, manual optical-band inversion, hard-ceiling +rejection, immediate mock transfer, board-code echo, square drive, and owner-service fail-safe +behavior. diff --git a/docs/features/stage-a-optical-waveform.md b/docs/features/stage-a-optical-waveform.md new file mode 100644 index 0000000..c0ab98b --- /dev/null +++ b/docs/features/stage-a-optical-waveform.md @@ -0,0 +1,125 @@ +# Stage-A Optical Waveform Drive + +- **Crate:** `plugins/stage-a-modulation` (`waveform.rs`) +- **Firmware:** `stage-a-controller` — `MOD wave=WARP` (`stimulus_mod::configureWarp`) +- **Status:** Analytic inversion, fed by a measured `V_null`/`Vπ` + ([Pockels transfer calibration](./stage-a-pockels-calibration.md)); a fully + measured LUT remains a documented follow-up +- **ADR:** [ADR 008](../adr/008-stage-a-optical-waveform-inversion.md) + +## Why + +The Pockels/PBS amplitude modulator has a `sin²` transfer, so a pure DAC sine +does **not** produce a sinusoidal *optical* target. On one monotonic lobe: + +```math +I(V) = I_\text{floor} + (I_\text{ceil}-I_\text{floor})\,\sin^2[\alpha (V - V_\text{null})], +\qquad \alpha = \frac{\pi}{2 V_\pi}. +``` + +To hit a chosen optical target the DAC must be pre-warped by inverting it: + +```math +u(t) = \frac{I_d(t)-I_\text{floor}}{I_\text{ceil}-I_\text{floor}},\qquad +V(u) = V_\text{null} + \frac{2 V_\pi}{\pi}\,\arcsin\!\sqrt{u}. +``` + +## Targets + +- **`OPTICAL_LOG_SINE`** (recommended A1 input): `ln I_d = ln I_g + (a/2)\sin\omega t`. + The event camera responds to changes in `ln I`, so this is the clean input. +- **`OPTICAL_LINEAR_SINE`**: `I_d = I_c(1 + m\sin\omega t)`, `m = \tanh(a/2)`. + +Both operate around an explicit operating point and are refused if their optical +maximum exceeds the lobe ceiling. `DAC_SINE` remains the pure-DAC sine. + +## Inversion parameters (settable — you do not need a rig to start) + +| Setting | Meaning | +|---|---| +| `V_null` | DAC code at the excitation minimum (`sin² = 0`) | +| `Vπ` | DAC-code quarter-wave distance from `V_null` to the excitation maximum | +| `a` | requested optical log-modulation depth `ln(I_max/I_min)` | +| `I_k` | operating illumination as a normalised lobe intensity `u_k ∈ (0,1]` | + +Get `V_null`/`Vπ` from a two-point check (code giving min light, code giving max +light on one lobe) or from nominal `Vπ ÷ driver volts-per-code`. The drive is +refused (never silently clamped) if `V_null + Vπ` overruns `0..4095`. + +### Fixed operating point `I_k`, swept depth `a` + +`I_k` is the geometric-mean point the modulation swings around: +`u(t) = u_k·exp[(a/2) sin ωt]` (log) or `u_k·(1 + m sin ωt)` (linear). **Hold +`I_k` fixed and sweep `a`** for one response curve. The drive is refused +(`Saturates`) when the peak `u_k·exp(a/2) > 1` — lower `I_k` or `a`. + +`CONST` is the exception because it does not modulate: it maps only `I_k` +through the inverse lobe and ignores `a`. For example, `V_null=1630`, +`Vπ=860` gives DAC `2490` at `I_k=1` and DAC `1685` at `I_k=0.01`. +Periodic modes still require the headroom above. Invalid setting changes are +rejected transactionally, so the UI retains the last applied value instead of +showing a target that the board never received. Photodiode RAW/EXCITATION mode +does not participate in this DAC calculation. + +### Drive method and hard ceiling + +Under `CALIBRATED`, `V_null`/`Vπ`/`I_k`/`a` define the operating band directly. +Under `MANUAL`, the Power + Min-threshold DAC endpoints are passed through the +forward `sin²` transfer and converted to the target law's effective `(I_k, a)`; +the same inverse-warp implementation then fills that band. + +Warp codes are absolute lobe codes and cannot be rescaled without distorting the +target. The plugin therefore **refuses** any drive whose peak exceeds the +always-visible `max_level` hard ceiling. Raise the max limit, or lower the +operating band / `I_k` / `a` / `Vπ`, to fit. + +### Modulation reference range + +For the current method the plugin reports the resolved DAC lower endpoint, +upper endpoint, constant hold code, and peak-to-peak swing. + +### Measured parameters (built) and the measured LUT (still future) + +`V_null`/`Vπ` are no longer typed in from a datasheet: the +[Pockels transfer calibration](./stage-a-pockels-calibration.md) sweeps settled +constant DAC codes, reads the photodiode level at each, and fits the lobe those +two parameters describe. The analytic `sin²` inversion above is unchanged — it is +now fed measured parameters. + +The fully measured **LUT** remains open: keep the swept `(code → optical level)` +table for one monotonic lobe and invert it directly instead of the analytic +form, dropping in behind the same `warp_table` interface and superseding +`V_null`/`Vπ` entirely. The calibration record already archives the points such +a table would need. + +## Wire form (firmware line limit) + +The command line is capped at 192 bytes, too small for a 256-code table, so the +plugin computes and validates the warp table locally (for the operator preview +and range guard) but sends the compact **parameters**: + +``` +MOD wave=WARP freq_mhz= target= a_milli= u_k_milli= v_null= v_pi= +``` + +The firmware rebuilds the identical 256-entry DAC table with the same formula +(`stimulus_mod::normalisedIntensity` + `dacForU`) and plays it back at the drive +frequency. A chunked **table upload** command is the natural extension for the +measured LUT. + +## Relationship to the measured `a` + +The requested `a` here is a *drive* target. The realised optical depth is always +the photodiode-measured `a` from the [photodiode plugin](./stage-a-photodiode.md) +(estimator geometry, rejected-complement corrected), never the commanded value. + +## Tests + +`cargo test -p augur-plugin-stage-a-modulation waveform` verifies both targets +stay in the DAC range, that feeding the warp table back through the `sin²` lobe +recovers the intended optical intensity, that the recovered log-contrast matches +the requested `a`, that a manual DAC band round-trips through +`OpticalDrive::from_dac_band`, and that invalid depth/inversion and lobe overruns +are refused. `cargo test -p stage-a-io mod_warp` covers the mock command surface. +The modulation-plugin tests also pin the full-lobe `CONST` values above and +verify that a rejected periodic `I_k` change cannot diverge from the board target. diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index ca688b4..122e6ff 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -14,7 +14,19 @@ plus the newest value. During a command-port acquisition the firmware mirrors th blocks here — every rate change or sample-index jump restarts the ring as a new segment, so the `index / rate` time base is always consistent. -Two modes: +## Phase-0 trigger overlay + +The firmware stamps a device-clock **`Marker` frame** (wire type 4) on the stream at every +modulation phase-0, in step with the J24 camera trigger. Because the chart is on the device +(Teensy) sample clock — not the camera clock — this stream marker is the correctly-aligned phase-0 +source (the camera `EXT_TRIGGER` belongs to A1's camera-clock analysis, not here). + +- **Show phase-0 trigger markers** (opt-in) overlays them as one toggleable vertical curve + ("phase-0 trigger") on the chart. +- The **modulation frequency is derived from the marker spacing** (`f = rate / mean marker gap`) and + shown in the status; the mock emits synthetic markers so the overlay works without hardware. + +## Modes - **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). - **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light @@ -32,6 +44,21 @@ Two modes: (`avg_sync_freq_hz`, e.g. the MOD drive frequency): window = `rate / f` samples, which makes the mean independent of the modulation phase instead of riding the waveform. +## Data (cache snapshot + disk recording) + +- The monitor cache always holds the last *N* seconds (`cache_s`). **Save cache + snapshot** writes it **once** as `pd_cache_.csv` + JSON sidecar. +- **Start recording** / **Stop recording** buttons tee every incoming sample + frame to `pd_rec_.pdq`; stopping writes the JSON sidecar. Both + buttons (and the snapshot) are disabled until a data directory is selected. +- All three are momentary buttons whose presses are forwarded from the UI + mirror to the live worker as monotonic press counters (`PressLatch`, ADR 010) + and act only on a press **edge**. The previous unguarded `save_snapshot` + handler fired on every host settings sync — one unwanted CSV per settings + change of *any* plugin — and the old `record` checkbox synced the mirror's + always-false state to the worker, so it could never stay recording. The + `record` boolean setting remains as a non-schema compatibility alias. + ## Contract - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the @@ -51,4 +78,5 @@ Two modes: jumps and rate changes, duration-bounded ring with aligned indexes, moving-average window derivation from the sync frequency, newest-window average, envelope decimation bounds and min ≤ mean ≤ max, raw rendering for short windows, excitation inversion, mock reader, settings -round-trips. +round-trips, the forwarded snapshot counter saving exactly once, and the record start/stop +buttons. diff --git a/docs/features/stage-a-pockels-calibration.md b/docs/features/stage-a-pockels-calibration.md new file mode 100644 index 0000000..d1a9eca --- /dev/null +++ b/docs/features/stage-a-pockels-calibration.md @@ -0,0 +1,214 @@ +# Stage-A Pockels Transfer Calibration + +- **Crate:** `plugins/stage-a-modulation` (`calibration.rs`) +- **Depends on:** `stage-a-photodiode` publishing `PhotodiodeStreamV1.level` +- **Status:** built +- **ADR:** [ADR 011](../adr/011-stage-a-pockels-transfer-calibration.md) +- **Knowledge base:** `methodology/pockels-waveform-linearisation.md` §4, + `setup/optical-path.md` + +## Why + +`V_null` and `Vπ` drive every calibrated waveform through the optical inversion +([Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md)), but they were +two bare number fields whose tooltip said *"measure it; do not trust nominal +Vπ"* — with no way to measure it. Nothing in the UI connected a DAC code to an +observed photodiode value, so the operator had to hand-sweep `CONST`, watch a +chart in another panel, and do the arithmetic by eye. + +## What it does + +One button. The modulation plugin steps settled `CONST` DAC codes across +`0..max_level` (49 points up, then the same 49 back down, ~20 s), reads the +photodiode level at each, and fits the lobe: + +```math +P(c) = p_0 + p_1 \sin^2\!\left[\frac{\pi (c - V_\text{null})}{2 V_\pi}\right] +``` + +The fit is then reviewed and applied by a second, explicit press. + +## Why the modulation plugin owns it + +It already owns `V_null`/`Vπ` and the DAC. The host broadcasts every plugin's +control snapshot to every plugin's inbox, so it reads photodiode levels **read +only** — no lease, no service command, no coordinating plugin, and no +photodiode recording. The photodiode simply needs to be connected. + +## Three things the physics forces + +**The detector port is an input, not a result.** `sin²` is symmetric about its +peak, so `(v, p_0, p_1)` and `(v + V_\pi, p_0 + p_1, -p_1)` fit the measured +curve *identically* — the data cannot say which extremum is zero excitation. + +The setting asks one observable question: *when the light reaching the sample +gets brighter, does the photodiode reading go up or down?* Stage-A's photodiode +sits on the PBS **reject** port and reads the light the sample does not get, +`I_pd = I_tot − I_exc`, so it falls as the sample brightens — and reads its +**maximum** at `V_null`. That is `REJECT PORT`, the default. `DIRECT` is for a +detector watching the sample beam itself. Declaring it wrong places `V_null` a +quarter wave off and runs the drive on the inverted branch. + +**The shape needs no dark measurement and no anchor.** `p_0` absorbs the dark +level and any DC offset; `p_1` absorbs the front-end gain. `V_null` and `Vπ` +are immune to both, which is why this procedure is one button and not a +protocol. + +**The absolute scale is *not* recoverable here.** On the reject port the +residual transmitted floor cannot be separated from the total-power anchor +`I_tot` (knowledge base §4.4). The detector level at the null is therefore +reported as a **lower bound** on `I_tot`, explicitly not as the anchor, and no +maximum achievable `a` is derived from it. Freezing a real anchor still needs a +transmitted-port power measurement. + +## How the fit works + +Because `sin²(x) = (1 − cos 2x)/2`, the model is a constant plus **one sinusoid +of period `2Vπ`**, and a sinusoid of known period is linear in its quadrature +components. So for each candidate `Vπ` the phase (hence `V_null`) and both +amplitudes come from a 3×3 linear solve, and only `Vπ` is searched: a +log-spaced scan over every period the sweep can resolve, then a golden-section +refine. + +Seeding the period from the measured extrema — the obvious approach — breaks on +exactly the sweeps that matter. At a realistic `Vπ ≈ 860` the DAC range holds +~2.4 lobes, so the global minimum and maximum can sit whole periods apart. + +Several nulls are valid when a sweep spans multiple lobes; the fit reports the +**lowest** one whose `[V_null, V_null + Vπ]` fits inside the max limit — least +voltage across the crystal, most headroom, and a rule the operator can predict. + +## Settling is proven, not timed + +Every published level carries `end_sample_index` and `sample_count` on the +device sample clock. A point is accepted only from a window that *began* at +least `SETTLE_SAMPLES` (2 000 ≈ 100 ms at 20 kSa/s) after its code was +commanded. No shared wall clock, no sleeps, immune to control-tick jitter. + +## The sweep owns the DAC while it runs + +`send_modulation` is silent for the duration. The host re-applies the *whole* +settings snapshot on every sync and most drive handlers push to the board +unconditionally, so without this the operator's armed waveform would be +re-armed on top of every commanded code — the board would play the armed drive +through the sweep, every point would read the same waveform-averaged level, and +the fit would report "the detector level did not change" on a bench where the +light was plainly modulating. Same shape as the automation-lease guard: a sweep +is another owner of the DAC. + +Settings changed mid-sweep are withheld, not rejected, and reach the board when +the sweep ends — the restore prefers the current drive and falls back to the +command captured at sweep start. + +## Interlocks + +The sweep refuses to start, and aborts if any becomes true mid-run, unless: +hardware effects are allowed on this instance, the command port is connected, +**no automation lease is held** (A1 must not be sweeping the drive at the same +time), no protocol is running, and a photodiode level is arriving. + +It always restores the pre-sweep drive — on completion, abort, stop press, +disconnect, or a stalled stream. A calibration sweep leaves the bench as it +found it. + +## Robustness: strays are dropped, the rest is a warning + +The fit runs twice. The first pass finds the period; points whose residual +exceeds **6× the median** absolute residual are then dropped and the fit is +repeated on what is left. The cut is on the median, not the mean or standard +deviation, because those are themselves dragged out by the very points being +looked for. `6 × median` is roughly 4σ for Gaussian noise, so ordinary scatter +survives untouched. + +This matters because of how the numbers actually behave on a bench. Measured on +a realistic small-signal sweep (90 mV span, `Vπ = 860`, 2.4 lobes): + +| Condition | Residual | Fitted `Vπ` | +|---|---|---| +| clean | 0.0 % | 860 | +| 5 mV noise | 3.1 % | 864 | +| 10 mV drift across the sweep | 3.2 % | 861 | +| 10 mV hysteresis | 5.5 % | 861 | +| **one stray point** | **9.9 %** | **863** | +| amplifier compressing the top of the range | 15.2 % | 1110 ✗ | + +A single bad sample inflates the residual fivefold while leaving `Vπ` accurate +to three codes — and it is invisible in the plot. That is why the residual +**warns and never blocks**: blocking on it withholds a good calibration for a +bad reason. A residual that stays high after rejection, with a visibly poor +overlay, is the real signal — and as the last row shows, it comes with a `Vπ` +that is wrong in a way the plot makes obvious. + +The fit is **never** applied automatically, and applying re-validates the +resulting drive: a calibration that cannot be armed is rolled back rather than +stored. Warnings surface as `Check:` lines in the status: + +| Warning | Meaning | +|---|---| +| residual > 5 % of the span | compare fit and points in the plot before trusting `Vπ` | +| points dropped | a couple is ordinary; a large share means the sweep is the problem | +| hysteresis > 5 % | the cell is drifting, or the settle time is too short | +| clipped points | the extremum they sit on is not where the fit thinks it is | + +There is no separate "lobe coverage" gate: `fit_transfer` already refuses a +sweep in which no full lobe fits inside the commandable range, so `Vπ` is always +measured rather than extrapolated by the time a fit exists. + +## The transfer-curve view + +A `LineSeriesWindow` host view, `Pockels transfer curve`: + +- **before any sweep** — the lobe the *configured* `V_null`/`Vπ` claim, on a + normalised `u` axis, with markers at `V_null` and `V_null + Vπ`. This works + with no hardware attached and is the answer to "what are these two numbers". +- **after a fit** — `measured ↑`, `measured ↓`, the fitted curve, and (while + they differ) the configured lobe on the fit's own scale, in detector volts. + +## Provenance + +Applying writes `pockels-.json` into the optional calibration folder +(points, fit, geometry, residual, hysteresis, and the anchor caveat) and sets +`ModulationStateV1.calibration_id`, so a consumer's sidecar can cite which +inversion produced a run's optical depth. Leaving the folder empty applies the +fit without archiving, and says so. + +## Dual-instance note + +The host renders `settings_schema()` from the **UI mirror**, which never owns +the device link, a lease, a sweep, or a fit. A `SettingKind::Button { enabled }` +may therefore only depend on state that is itself a setting — anything else is +invisible to the instance that draws it and disables the button forever. The +calibration buttons gate on "the operator asked to connect"; every real +interlock is enforced on the worker and reported in the status lines, which the +host does take from the worker. + +## Settings + +| Key | Meaning | +|---|---| +| `detector_geometry` | which PBS port the photodiode watches (`REJECT PORT` default) | +| `calibrate` | measure the transfer curve; press again to abort | +| `calibrate_apply` | write the reviewed fit into `V_null`/`Vπ` | +| `calibration_dir` | optional archive folder for the calibration record | + +`V_null`/`Vπ` remain directly editable as the manual override. + +## Verification + +- `calibration.rs` unit tests recover a known lobe from **both** ports, across + a multi-lobe sweep, and with a null at code 0; they check the geometry input + selects between the two equivalent representations, and that flat sweeps, + short sweeps, and out-of-range lobes are refused. +- An end-to-end test runs the sweep against the mock board, synthesizing the + light the reject-port detector *would* report for whatever code the board is + actually holding — ground truth for commanding, settle gating, point + collection, the fit, and the drive restore. + +## Limits + +- Analytic `sin²` inversion, not a measured LUT (the knowledge base's eventual + target); the calibration record stores the points a LUT would need. +- Dark level and the total-power anchor remain separate measurements. +- Static transfer only. A static calibration must never be used to correct + dynamic roll-off — that would manufacture the Bode curve A1 measures + (knowledge base "Gotchas"). diff --git a/docs/features/stage-a.md b/docs/features/stage-a.md index ab9167f..e3bfbe2 100644 --- a/docs/features/stage-a.md +++ b/docs/features/stage-a.md @@ -1,7 +1,7 @@ # Stage-A Bench Stack -- **Status:** Simplified two-plugin setup (2026-07-15, ADR 006) -- **Firmware:** `stage-a-controller` 0.3.0 (Teensy 4.1 on Hermit V2r1, `USB_DUAL_SERIAL`) +- **Status:** Two persistent owners plus orchestrated experiment workflows (ADR 007) +- **Firmware:** `stage-a-controller` 0.4.0+ (Teensy 4.1 on Hermit V2r1, `USB_DUAL_SERIAL`) ## Current shape @@ -10,22 +10,26 @@ The Teensy enumerates as **two** USB serial ports, and each is owned by exactly | Port | Content | Owner | |---|---|---| | command port (first) | v1 ASCII commands + PDA1 binary frames | [`stage-a-modulation`](./stage-a-modulation.md) | -| stream port (second) | free-running `PD code=… n=… t_ms=…` lines, 50 Hz | [`stage-a-photodiode`](./stage-a-photodiode.md) | +| stream port (second) | free-running PDA1 `SamplesU16` frames, 20 kSa/s default | [`stage-a-photodiode`](./stage-a-photodiode.md) | -- **`stage-a-modulation`** — capped power slider + constant/sine/square drive of the laser - modulation input (J23), transferred to the Teensy immediately; shows the board-reported DAC - code. Firmware output is set-and-hold; "Output OFF" is the explicit stop. +- **`stage-a-modulation`** — Manual/Calibrated operating-band selection plus five independent + waveform modes under one hard DAC ceiling, transferred to J23 immediately; shows the resolved + band and board-reported DAC code. Firmware output is set-and-hold; automation uses an explicit + `SafeOff` operation. - **`stage-a-photodiode`** — live readout of SMA5/pin 18/A4, raw or inverted to excitation power `I_exc = I_tot − I_pd` against a user-set reference. - **`stage-a-io`** (shared non-plugin library) — PDA1 wire format, typed client with idempotent retries, bounded I/O worker, and a firmware-faithful mock (including the 0.3.0 `MOD` verb). - The estimator/pdq/sidecar modules are retained for the future A1–A3 experiment plugins. + The photodiode owner uses the parser/PDQ modules; experiment plugins may use + hardware-free readers/analysis but never open the ports. +- **`stage-a-a1`** — orchestrates both owner services and camera recording; it + never opens a Teensy port or writes PDQ directly. Architecture: ADR 007. ## History -The earlier commissioning stack (`stage-a-monitor`, `stage-a-funcgen`, `stage-a-a1` — device +The earlier commissioning stack (`stage-a-monitor`, `stage-a-funcgen`, old `stage-a-a1` — device monitor with calibrated contrast, waveform familiarisation, and the A1 minimum-depth Bode sweep) was removed on 2026-07-15 as too complex for the current bench stage (ADR 006). It remains in git -history; the experiment plugins will be rebuilt on the simplified stack when the bench needs -them. Device-ownership and safety rules: ADR 005 (one owner per port, fail-closed effects gate) -as amended by ADR 006. +history; the new A1 implementation uses different statistics and host-routed +orchestration. Device-ownership and safety rules: ADR 005 as amended by ADR 006 +and ADR 007. diff --git a/plugins/stage-a-a1/Cargo.toml b/plugins/stage-a-a1/Cargo.toml new file mode 100644 index 0000000..4795fbe --- /dev/null +++ b/plugins/stage-a-a1/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "augur-plugin-stage-a-a1" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +description = "Stage-A A1 workflow orchestrator and pure minimum-depth analysis core" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +augur-plugin-api.workspace = true +serde.workspace = true +serde_json.workspace = true +stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } +toml = "0.8" + +[lints.rust] +unsafe_code = "forbid" diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md new file mode 100644 index 0000000..50d98a6 --- /dev/null +++ b/plugins/stage-a-a1/README.md @@ -0,0 +1,53 @@ +# Stage-A A1 Analysis + +`stage-a-a1` is the Stage-A **recording coordinator** plus two live sanity quicklooks. One button +records the camera **RAW** stream and the photodiode **PDQ** stream together for a fixed duration, +groups them under a per-`(I_k, f)` measurement id, and writes an A1 config sidecar (`.toml`) linking +the files with the modulation settings, the measured modulation depth `a`, the ROI, and the trigger +info needed to reproduce and analyse the run offline. A second button, **Start sweep**, repeats that +per amplitude: it leases the modulation owner, retargets the armed calibrated drive to each `a` in +`[Sweep min a, Sweep max a]`, waits for the photodiode-measured `a` to settle, and records every +point (`…_pNN`). Outside the leased sweep A1 owns no hardware and never drives the Teensy — arm the +optical drive in the modulation plugin; A1 only reads its published settings. + +## Recording + +- **Output folder** — where the A1 config sidecar is written (recommended shared experiment root). +- **Measurement id** — one per `(I_k, f)` pair; auto-generated default, editable, or press **New id**. +- **Duration (s)** — each recording auto-stops and finalizes after this. +- **Start recording** — starts camera RAW, then connects/leases the photodiode and starts PDQ; + the timer begins after both acknowledge. It auto-finalizes PDQ first, camera second, then writes + the sidecar. **Stop** saves the current recording early (and aborts a running sweep). +- **Start sweep** — records **Sweep points (count)** amplitudes spanning `[Sweep min a, Sweep max a]` + (min > 0): per point it renews the modulation lease, issues `SetOpticalDepth`, waits for the + measured `a` to hold the target for **Sweep settle (s)** (30 s cap, then records anyway), and runs + one normal recording. Sidecars carry `sweep.requested_a` / `point_index` / `point_total`. +- The record/sweep buttons are disabled until an output folder is selected. + +Files share an `_` stem: `/_.raw` (camera, under the host output root), +`/__pd.pdq` + `.json` (photodiode, under its data root), and +`/__config.toml` (A1, under the chosen folder). Point all three roots at the same +experiment directory to co-locate everything. The host also writes its own `.toml` next to the +RAW with the camera biases/ROI; the A1 sidecar cross-references it. + +## Live quicklooks + +- **Rolling half-period response** `S_p(t) = N_p(t−T/2, t] / N_valid` — events per valid pixel in the + trailing half-cycle, ON and OFF. A live "are events appearing, is the ON/OFF timing sane?" check. +- **Response probability** `q_p` — fraction of valid pixel-cycles that fire at least once in the + ON/OFF phase window (each pixel-cycle counts once, unlike `S_p`). The windows come from the row's + **pilot** when one has been recorded (frozen and held across the row), otherwise auto-detected + from the trigger-anchored fold (each grows out from its histogram peak to the window floor, + default 10 % of peak). `Record pilot` / `Record background` (in the Recording section) capture the + frozen windows and the floor `q0` into the measurement folder and are auto-reloaded when you + return to that folder + id. Record one point per amplitude vs the photodiode-measured `a`. The + authoritative `q_p(a, f)` fit is computed **offline** from the recordings; this is a quicklook. + +The period `T` comes from the firmware phase-0 `EXT_TRIGGER` marker spacing (the trigger *defines* +the frequency), falling back to the modulation plugin's acknowledged waveform. The ROI and masked +pixels come from the augur-rs camera config. + +See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the full brief, +[ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, and +[docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the +planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/plugin.toml b/plugins/stage-a-a1/plugin.toml new file mode 100644 index 0000000..4515f43 --- /dev/null +++ b/plugins/stage-a-a1/plugin.toml @@ -0,0 +1,9 @@ +id = "stage-a.a1" +name = "Stage-A A1 Analysis" +version = "0.3.0" +description = "Stage-A A1 recording coordinator: one-button synchronized camera .raw + photodiode .pdq recording with a config sidecar, plus live rolling-response and response-probability quicklooks." +domain = "stage-a" +library = "augur_plugin_stage_a_a1" +phase = "raw_events" +min_augur_version = "1.0.0" +host_commands = ["start_recording", "stop_recording"] diff --git a/plugins/stage-a-a1/src/lib.rs b/plugins/stage-a-a1/src/lib.rs new file mode 100644 index 0000000..b67952b --- /dev/null +++ b/plugins/stage-a-a1/src/lib.rs @@ -0,0 +1,14 @@ +//! Pure scientific and workflow core for the Stage-A A1 experiment. +//! +//! This crate intentionally contains no serial transport, Teensy client, or +//! PDQ writer. Hardware ownership remains with the Stage-A modulation and +//! photodiode plugins; this code only validates and analyses immutable inputs. + +pub mod phase; +pub mod rates; +pub mod response_curve; +mod runtime; +pub mod types; + +pub use runtime::StageAA1Plugin; +pub use types::{CameraEvent, Polarity}; diff --git a/plugins/stage-a-a1/src/phase.rs b/plugins/stage-a-a1/src/phase.rs new file mode 100644 index 0000000..0db0032 --- /dev/null +++ b/plugins/stage-a-a1/src/phase.rs @@ -0,0 +1,382 @@ +//! EXT_TRIGGER marker validation and camera-clock phase folding. + +use crate::types::{CameraEvent, Polarity}; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct MarkerValidationConfig { + pub expected_frequency_hz: f64, + pub frequency_tolerance_fraction: f64, + pub max_period_jitter_fraction: f64, + /// Expected complete cycles, when the acquisition declared one. + pub expected_cycles: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MarkerValidation { + pub cycle_count: usize, + pub measured_frequency_hz: f64, + pub mean_period_us: f64, + pub max_period_jitter_fraction: f64, + pub first_marker_us: u64, + pub last_marker_us: u64, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum MarkerError { + InvalidConfiguration(&'static str), + TooFewMarkers { + count: usize, + }, + NonIncreasing { + index: usize, + }, + CycleCount { + expected: usize, + actual: usize, + }, + FrequencyOutOfTolerance { + expected_hz: f64, + measured_hz: f64, + tolerance_fraction: f64, + }, + JitterOutOfTolerance { + measured_fraction: f64, + tolerance_fraction: f64, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FoldedEvent { + pub timestamp_us: u64, + pub x: u16, + pub y: u16, + pub polarity: Polarity, + pub cycle_index: usize, + /// Circular phase in `[0, 1)`. + pub phase: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseFold { + pub markers_us: Vec, + pub validation: MarkerValidation, + pub events: Vec, + pub events_outside_complete_cycles: usize, +} + +impl PhaseFold { + pub fn phase_at(&self, timestamp_us: u64) -> f64 { + let period_us = self.validation.mean_period_us; + (timestamp_us.saturating_sub(self.validation.first_marker_us) as f64 / period_us) + .rem_euclid(1.0) + } +} + +pub fn validate_markers( + markers_us: &[u64], + config: MarkerValidationConfig, +) -> Result { + if !config.expected_frequency_hz.is_finite() || config.expected_frequency_hz <= 0.0 { + return Err(MarkerError::InvalidConfiguration( + "expected frequency must be finite and positive", + )); + } + if !config.frequency_tolerance_fraction.is_finite() + || config.frequency_tolerance_fraction < 0.0 + || !config.max_period_jitter_fraction.is_finite() + || config.max_period_jitter_fraction < 0.0 + { + return Err(MarkerError::InvalidConfiguration( + "marker tolerances must be finite and non-negative", + )); + } + if markers_us.len() < 2 { + return Err(MarkerError::TooFewMarkers { + count: markers_us.len(), + }); + } + + let mut periods = Vec::with_capacity(markers_us.len() - 1); + for (index, pair) in markers_us.windows(2).enumerate() { + if pair[1] <= pair[0] { + return Err(MarkerError::NonIncreasing { index: index + 1 }); + } + periods.push((pair[1] - pair[0]) as f64); + } + + let cycle_count = periods.len(); + if let Some(expected) = config.expected_cycles { + if cycle_count != expected { + return Err(MarkerError::CycleCount { + expected, + actual: cycle_count, + }); + } + } + let mean_period_us = periods.iter().sum::() / cycle_count as f64; + let measured_frequency_hz = 1_000_000.0 / mean_period_us; + let frequency_error = ((measured_frequency_hz - config.expected_frequency_hz) + / config.expected_frequency_hz) + .abs(); + if frequency_error > config.frequency_tolerance_fraction { + return Err(MarkerError::FrequencyOutOfTolerance { + expected_hz: config.expected_frequency_hz, + measured_hz: measured_frequency_hz, + tolerance_fraction: config.frequency_tolerance_fraction, + }); + } + + let max_period_jitter_fraction = periods + .iter() + .map(|period| ((period - mean_period_us) / mean_period_us).abs()) + .fold(0.0_f64, f64::max); + if max_period_jitter_fraction > config.max_period_jitter_fraction { + return Err(MarkerError::JitterOutOfTolerance { + measured_fraction: max_period_jitter_fraction, + tolerance_fraction: config.max_period_jitter_fraction, + }); + } + + Ok(MarkerValidation { + cycle_count, + measured_frequency_hz, + mean_period_us, + max_period_jitter_fraction, + first_marker_us: markers_us[0], + last_marker_us: *markers_us.last().expect("at least two markers"), + }) +} + +/// Folds events against a free-running modulation period, with the phase +/// origin placed at the first event. This is the "phase-0 unanchored" path +/// used until a hardware `EXT_TRIGGER` reaches the camera: bins are relative +/// to the first event, not tied to the drive waveform. Only events inside the +/// whole-cycle span are retained so the rate normalisation matches +/// `cycle_count`. Returns `None` when the period is invalid or the window does +/// not cover at least one whole cycle. +pub fn fold_events_free_running(events: &[CameraEvent], period_us: f64) -> Option { + if !period_us.is_finite() || period_us <= 0.0 || events.is_empty() { + return None; + } + let first = events.iter().map(|event| event.timestamp_us).min()?; + let last = events.iter().map(|event| event.timestamp_us).max()?; + let cycle_count = ((last.saturating_sub(first)) as f64 / period_us).floor() as usize; + if cycle_count == 0 { + return None; + } + + let mut folded = Vec::with_capacity(events.len()); + let mut outside = 0; + for event in events { + let cycles = event.timestamp_us.saturating_sub(first) as f64 / period_us; + let cycle_index = cycles.floor() as usize; + if cycle_index >= cycle_count { + outside += 1; + continue; + } + folded.push(FoldedEvent { + timestamp_us: event.timestamp_us, + x: event.x, + y: event.y, + polarity: event.polarity, + cycle_index, + phase: cycles.fract(), + }); + } + + Some(PhaseFold { + markers_us: Vec::new(), + validation: MarkerValidation { + cycle_count, + measured_frequency_hz: 1_000_000.0 / period_us, + mean_period_us: period_us, + max_period_jitter_fraction: 0.0, + first_marker_us: first, + last_marker_us: first + (cycle_count as f64 * period_us).round() as u64, + }, + events: folded, + events_outside_complete_cycles: outside, + }) +} + +pub fn fold_events( + events: &[CameraEvent], + markers_us: &[u64], + config: MarkerValidationConfig, +) -> Result { + let validation = validate_markers(markers_us, config)?; + let mut folded = Vec::with_capacity(events.len()); + let mut outside = 0; + + for event in events { + let cycle_index = match markers_us.binary_search(&event.timestamp_us) { + Ok(index) if index + 1 < markers_us.len() => index, + Ok(_) => { + outside += 1; + continue; + } + Err(0) => { + outside += 1; + continue; + } + Err(index) if index < markers_us.len() => index - 1, + Err(_) => { + outside += 1; + continue; + } + }; + let start = markers_us[cycle_index]; + let end = markers_us[cycle_index + 1]; + let phase = (event.timestamp_us - start) as f64 / (end - start) as f64; + folded.push(FoldedEvent { + timestamp_us: event.timestamp_us, + x: event.x, + y: event.y, + polarity: event.polarity, + cycle_index, + phase, + }); + } + + Ok(PhaseFold { + markers_us: markers_us.to_vec(), + validation, + events: folded, + events_outside_complete_cycles: outside, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> MarkerValidationConfig { + MarkerValidationConfig { + expected_frequency_hz: 1_000.0, + frequency_tolerance_fraction: 0.01, + max_period_jitter_fraction: 0.02, + expected_cycles: Some(3), + } + } + + #[test] + fn validates_and_folds_against_camera_clock_markers() { + let markers = [10_000, 11_000, 12_000, 13_000]; + let events = [ + CameraEvent { + timestamp_us: 10_250, + x: 1, + y: 2, + polarity: Polarity::On, + }, + CameraEvent { + timestamp_us: 11_750, + x: 3, + y: 4, + polarity: Polarity::Off, + }, + CameraEvent { + timestamp_us: 13_000, + x: 0, + y: 0, + polarity: Polarity::On, + }, + ]; + let fold = fold_events(&events, &markers, config()).expect("valid markers"); + assert_eq!(fold.validation.cycle_count, 3); + assert_eq!(fold.events.len(), 2); + assert_eq!(fold.events_outside_complete_cycles, 1); + assert_eq!(fold.events[0].cycle_index, 0); + assert!((fold.events[0].phase - 0.25).abs() < 1e-12); + assert_eq!(fold.events[1].cycle_index, 1); + assert!((fold.events[1].phase - 0.75).abs() < 1e-12); + } + + #[test] + fn rejects_marker_count_frequency_and_jitter_mismatches() { + let mut wrong_count = config(); + wrong_count.expected_cycles = Some(4); + assert!(matches!( + validate_markers(&[0, 1_000, 2_000, 3_000], wrong_count), + Err(MarkerError::CycleCount { .. }) + )); + + assert!(matches!( + validate_markers(&[0, 2_000, 4_000, 6_000], config()), + Err(MarkerError::FrequencyOutOfTolerance { .. }) + )); + + assert!(matches!( + validate_markers(&[0, 1_000, 2_100, 3_000], config()), + Err(MarkerError::JitterOutOfTolerance { .. }) + )); + } + + #[test] + fn free_running_fold_bins_relative_to_first_event() { + // Period 1000 us; three whole cycles from the first event at 500 us. + let events = [ + CameraEvent { + timestamp_us: 500, + x: 0, + y: 0, + polarity: Polarity::On, + }, + CameraEvent { + timestamp_us: 750, + x: 0, + y: 0, + polarity: Polarity::Off, + }, + CameraEvent { + timestamp_us: 1_750, + x: 0, + y: 0, + polarity: Polarity::On, + }, + // Beyond the last whole cycle -> excluded. + CameraEvent { + timestamp_us: 4_000, + x: 0, + y: 0, + polarity: Polarity::On, + }, + ]; + let fold = fold_events_free_running(&events, 1_000.0).expect("one whole cycle"); + assert_eq!(fold.validation.cycle_count, 3); + assert_eq!(fold.events.len(), 3); + assert_eq!(fold.events_outside_complete_cycles, 1); + assert!((fold.events[0].phase - 0.0).abs() < 1e-12); + assert!((fold.events[1].phase - 0.25).abs() < 1e-12); + assert_eq!(fold.events[1].cycle_index, 0); + assert_eq!(fold.events[2].cycle_index, 1); + assert!((fold.events[2].phase - 0.25).abs() < 1e-12); + } + + #[test] + fn free_running_fold_needs_one_whole_cycle() { + let events = [ + CameraEvent { + timestamp_us: 0, + x: 0, + y: 0, + polarity: Polarity::On, + }, + CameraEvent { + timestamp_us: 400, + x: 0, + y: 0, + polarity: Polarity::On, + }, + ]; + assert!(fold_events_free_running(&events, 1_000.0).is_none()); + } + + #[test] + fn rejects_non_monotonic_markers() { + assert_eq!( + validate_markers(&[0, 1_000, 999, 2_000], config()), + Err(MarkerError::NonIncreasing { index: 2 }) + ); + } +} diff --git a/plugins/stage-a-a1/src/rates.rs b/plugins/stage-a-a1/src/rates.rs new file mode 100644 index 0000000..605a910 --- /dev/null +++ b/plugins/stage-a-a1/src/rates.rs @@ -0,0 +1,294 @@ +//! Phase-bin event rates and rolling half-period operator quicklooks. + +use crate::phase::PhaseFold; +use crate::types::Polarity; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RateLayer { + pub count: u64, + /// Events per valid pixel per second. + pub rate_per_pixel_s: f64, + /// Poisson standard error in the same units as `rate_per_pixel_s`. + pub standard_error: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseRateBin { + pub phase_start: f64, + pub phase_end: f64, + pub run: RateLayer, + pub background: Option, + /// Run minus background. Negative values are intentionally preserved. + pub net_rate_per_pixel_s: Option, + /// Independent Poisson uncertainty propagated in quadrature. + pub net_standard_error: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PolarityPhaseRates { + pub polarity: Polarity, + pub valid_pixels: usize, + pub run_cycles: usize, + pub background_cycles: Option, + pub bins: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PhaseRateSet { + pub on: PolarityPhaseRates, + pub off: PolarityPhaseRates, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum RateError { + ZeroValidPixels, + InvalidBinCount, + EmptyCycles, +} + +pub fn phase_bin_rates( + run: &PhaseFold, + background: Option<&PhaseFold>, + valid_pixels: usize, + bin_count: usize, +) -> Result { + if valid_pixels == 0 { + return Err(RateError::ZeroValidPixels); + } + if bin_count == 0 { + return Err(RateError::InvalidBinCount); + } + if run.validation.cycle_count == 0 + || background.is_some_and(|fold| fold.validation.cycle_count == 0) + { + return Err(RateError::EmptyCycles); + } + + Ok(PhaseRateSet { + on: rates_for_polarity(run, background, valid_pixels, bin_count, Polarity::On), + off: rates_for_polarity(run, background, valid_pixels, bin_count, Polarity::Off), + }) +} + +fn rates_for_polarity( + run: &PhaseFold, + background: Option<&PhaseFold>, + valid_pixels: usize, + bin_count: usize, + polarity: Polarity, +) -> PolarityPhaseRates { + let mut run_counts = vec![0_u64; bin_count]; + let mut background_counts = vec![0_u64; bin_count]; + for event in run.events.iter().filter(|event| event.polarity == polarity) { + run_counts[phase_bin(event.phase, bin_count)] += 1; + } + if let Some(background) = background { + for event in background + .events + .iter() + .filter(|event| event.polarity == polarity) + { + background_counts[phase_bin(event.phase, bin_count)] += 1; + } + } + + let run_bin_s = run.validation.mean_period_us / 1_000_000.0 / bin_count as f64; + let run_exposure = valid_pixels as f64 * run.validation.cycle_count as f64 * run_bin_s; + let background_exposure = background.map(|fold| { + valid_pixels as f64 + * fold.validation.cycle_count as f64 + * (fold.validation.mean_period_us / 1_000_000.0 / bin_count as f64) + }); + + let bins = (0..bin_count) + .map(|index| { + let run_layer = poisson_layer(run_counts[index], run_exposure); + let background_layer = background_exposure + .map(|exposure| poisson_layer(background_counts[index], exposure)); + let (net, net_error) = background_layer.map_or((None, None), |background| { + ( + Some(run_layer.rate_per_pixel_s - background.rate_per_pixel_s), + Some( + (run_layer.standard_error.powi(2) + background.standard_error.powi(2)) + .sqrt(), + ), + ) + }); + PhaseRateBin { + phase_start: index as f64 / bin_count as f64, + phase_end: (index + 1) as f64 / bin_count as f64, + run: run_layer, + background: background_layer, + net_rate_per_pixel_s: net, + net_standard_error: net_error, + } + }) + .collect(); + + PolarityPhaseRates { + polarity, + valid_pixels, + run_cycles: run.validation.cycle_count, + background_cycles: background.map(|fold| fold.validation.cycle_count), + bins, + } +} + +fn phase_bin(phase: f64, bin_count: usize) -> usize { + ((phase.rem_euclid(1.0) * bin_count as f64).floor() as usize).min(bin_count - 1) +} + +fn poisson_layer(count: u64, exposure_pixel_s: f64) -> RateLayer { + RateLayer { + count, + rate_per_pixel_s: count as f64 / exposure_pixel_s, + standard_error: (count as f64).sqrt() / exposure_pixel_s, + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RollingResponsePoint { + pub timestamp_us: u64, + /// Events in `(t - T/2, t]` per valid pixel. + pub run_per_pixel: f64, + /// Integral of the periodic phase-resolved background rate, when enabled. + pub background_per_pixel: Option, + pub net_per_pixel: Option, +} + +pub fn rolling_half_period_response( + run: &PhaseFold, + polarity: Polarity, + valid_pixels: usize, + sample_times_us: &[u64], + background_model: Option<&PolarityPhaseRates>, +) -> Result, RateError> { + if valid_pixels == 0 { + return Err(RateError::ZeroValidPixels); + } + let half_period_us = run.validation.mean_period_us / 2.0; + let period_s = run.validation.mean_period_us / 1_000_000.0; + Ok(sample_times_us + .iter() + .map(|×tamp_us| { + let window_start = timestamp_us as f64 - half_period_us; + let count = run + .events + .iter() + .filter(|event| { + event.polarity == polarity + && event.timestamp_us as f64 > window_start + && event.timestamp_us <= timestamp_us + }) + .count(); + let run_per_pixel = count as f64 / valid_pixels as f64; + let background_per_pixel = background_model.map(|model| { + let start_phase = run.phase_at(timestamp_us.saturating_sub(half_period_us as u64)); + integrate_periodic_rates(model, start_phase, 0.5) * period_s + }); + RollingResponsePoint { + timestamp_us, + run_per_pixel, + background_per_pixel, + net_per_pixel: background_per_pixel.map(|bg| run_per_pixel - bg), + } + }) + .collect()) +} + +/// Integrates rates over a circular phase span and returns rate × phase. +fn integrate_periodic_rates(model: &PolarityPhaseRates, start_phase: f64, phase_span: f64) -> f64 { + let mut total = 0.0; + let start = start_phase.rem_euclid(1.0); + let end = start + phase_span; + for bin in &model.bins { + for offset in [0.0, 1.0] { + let bin_start = bin.phase_start + offset; + let bin_end = bin.phase_end + offset; + let overlap = (end.min(bin_end) - start.max(bin_start)).max(0.0); + let layer = bin.background.unwrap_or(bin.run); + total += overlap * layer.rate_per_pixel_s; + } + } + total +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::phase::{fold_events, MarkerValidationConfig}; + use crate::types::CameraEvent; + + fn fold(events: &[CameraEvent]) -> PhaseFold { + fold_events( + events, + &[0, 1_000, 2_000], + MarkerValidationConfig { + expected_frequency_hz: 1_000.0, + frequency_tolerance_fraction: 0.0, + max_period_jitter_fraction: 0.0, + expected_cycles: Some(2), + }, + ) + .unwrap() + } + + fn event(timestamp_us: u64, polarity: Polarity) -> CameraEvent { + CameraEvent { + timestamp_us, + x: 0, + y: 0, + polarity, + } + } + + #[test] + fn computes_raw_background_and_negative_net_rates_per_polarity() { + let run = fold(&[ + event(100, Polarity::On), + event(1_100, Polarity::On), + event(600, Polarity::Off), + ]); + let background = fold(&[ + event(100, Polarity::On), + event(200, Polarity::On), + event(1_100, Polarity::On), + event(1_200, Polarity::On), + ]); + let rates = phase_bin_rates(&run, Some(&background), 10, 2).unwrap(); + let on_first = &rates.on.bins[0]; + assert_eq!(on_first.run.count, 2); + assert_eq!(on_first.background.unwrap().count, 4); + assert!(on_first.net_rate_per_pixel_s.unwrap() < 0.0); + assert!(on_first.net_standard_error.unwrap() > 0.0); + assert_eq!(rates.off.bins[1].run.count, 1); + } + + #[test] + fn rolling_quicklook_uses_open_left_closed_right_window_and_background_integral() { + let run = fold(&[ + event(500, Polarity::On), + event(750, Polarity::On), + event(1_000, Polarity::On), + ]); + let background = fold(&[ + event(100, Polarity::On), + event(600, Polarity::On), + event(1_100, Polarity::On), + event(1_600, Polarity::On), + ]); + let background_rates = phase_bin_rates(&run, Some(&background), 1, 2).unwrap(); + let points = rolling_half_period_response( + &run, + Polarity::On, + 1, + &[1_000], + Some(&background_rates.on), + ) + .unwrap(); + // Event at exactly t-T/2 is excluded; 750 and 1000 are included. + assert_eq!(points[0].run_per_pixel, 2.0); + assert!((points[0].background_per_pixel.unwrap() - 1.0).abs() < 1e-12); + assert!((points[0].net_per_pixel.unwrap() - 1.0).abs() < 1e-12); + } +} diff --git a/plugins/stage-a-a1/src/response_curve.rs b/plugins/stage-a-a1/src/response_curve.rs new file mode 100644 index 0000000..b4540a2 --- /dev/null +++ b/plugins/stage-a-a1/src/response_curve.rs @@ -0,0 +1,293 @@ +//! Auto-windowed Bernoulli response probability `q_p(a, f)`. +//! +//! With the firmware phase-0 `EXT_TRIGGER` anchoring the camera phase, ON and OFF +//! events fall in opposite half-cycles, so the ON/OFF phase windows can be found +//! directly from the current fold — no separate bright "pilot" capture is needed. +//! +//! For each polarity we anchor a window on its phase-histogram peak and grow it +//! outward while the histogram stays above a floor (a fraction of the peak) **and** +//! that polarity still dominates the opposite one. The window therefore ends out in +//! the opposite half-cycle, where the polarity's events have died away, and can +//! never bleed into the other polarity's cluster. +//! +//! The response probability is then, per pixel `i` and cycle `c`: +//! +//! ```text +//! z_{i,c,p} = 1 if pixel i fires at least once in W_p during cycle c, else 0 +//! q_p(a,f) = (1 / (N_valid · M)) · Σ_i Σ_c z_{i,c,p} +//! ``` +//! +//! computed independently for ON and OFF, where `M` is the number of complete +//! valid cycles and `N_valid` is the ROI minus masked pixels. +//! +//! This is the **live quicklook** definition. The authoritative `q_p(a, f)` fit +//! freezes the windows once (from the brightest recording) and applies them to all +//! amplitudes offline — auto-windowing per fold is deliberately not amplitude-frozen. + +use std::collections::HashSet; + +use crate::phase::PhaseFold; +use crate::types::Polarity; + +/// Phase-histogram resolution used for window detection. +pub const HIST_BINS: usize = 64; + +/// Circular phase window `[start, end)` in cycle fraction. When `start > end` +/// the window wraps past 1.0. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PhaseWindow { + pub start: f64, + pub end: f64, +} + +impl PhaseWindow { + pub fn contains(&self, phase: f64) -> bool { + let p = phase.rem_euclid(1.0); + if self.start <= self.end { + p >= self.start && p < self.end + } else { + p >= self.start || p < self.end + } + } +} + +/// Region of interest in pixel coordinates; `x1`/`y1` are exclusive. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Roi { + pub x0: u16, + pub y0: u16, + pub x1: u16, + pub y1: u16, +} + +impl Roi { + pub fn contains(&self, x: u16, y: u16) -> bool { + x >= self.x0 && x < self.x1 && y >= self.y0 && y < self.y1 + } + + pub fn area(&self) -> usize { + usize::from(self.x1.saturating_sub(self.x0)) * usize::from(self.y1.saturating_sub(self.y0)) + } +} + +/// One recorded response-curve point. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ResponsePoint { + pub measured_a: f64, + pub q_on: f64, + pub q_off: f64, + pub cycles: usize, + pub valid_pixels: usize, +} + +/// ON/OFF phase histogram of a fold. +pub fn phase_histogram(fold: &PhaseFold, polarity: Polarity) -> Vec { + let mut histogram = vec![0.0; HIST_BINS]; + for event in fold + .events + .iter() + .filter(|event| event.polarity == polarity) + { + let phase = event.phase.rem_euclid(1.0); + let bin = ((phase * HIST_BINS as f64) as usize).min(HIST_BINS - 1); + histogram[bin] += 1.0; + } + histogram +} + +/// Grows a circular window out from `hist`'s peak while the peak-relative floor is +/// met and this polarity keeps dominating `other`. Returns `None` on an empty +/// histogram. +fn grow_window(hist: &[f64], other: &[f64], floor_fraction: f64) -> Option { + let bins = hist.len(); + let peak = hist.iter().copied().fold(0.0_f64, f64::max); + if peak <= 0.0 || bins == 0 { + return None; + } + let floor = peak * floor_fraction; + let peak_bin = hist + .iter() + .enumerate() + .max_by(|a, b| a.1.total_cmp(b.1)) + .map(|(index, _)| index)?; + + // A bin belongs to this window when it clears the floor and this polarity is + // at least as strong as the opposite one there. + let keep = |index: usize| hist[index] >= floor && hist[index] >= other[index]; + + // The peak anchors the window; grow right then left until a bin fails. + let mut right = peak_bin; + for step in 1..bins { + let index = (peak_bin + step) % bins; + if keep(index) { + right = index; + } else { + break; + } + } + let mut left = peak_bin; + for step in 1..bins { + let index = (peak_bin + bins - step) % bins; + if keep(index) { + left = index; + } else { + break; + } + } + + Some(PhaseWindow { + start: left as f64 / bins as f64, + end: ((right + 1) % bins) as f64 / bins as f64, + }) +} + +/// Detects the ON and OFF phase windows directly from a fold's histograms. +pub fn auto_windows(fold: &PhaseFold, floor_fraction: f64) -> Option<(PhaseWindow, PhaseWindow)> { + let on = phase_histogram(fold, Polarity::On); + let off = phase_histogram(fold, Polarity::Off); + let window_on = grow_window(&on, &off, floor_fraction)?; + let window_off = grow_window(&off, &on, floor_fraction)?; + Some((window_on, window_off)) +} + +/// Computes the ON/OFF Bernoulli response probabilities for one fold against the +/// given windows. `masked` holds pixels excluded inside the ROI. Returns `None` +/// when there are no complete cycles or no valid pixels. +pub fn response_probability( + fold: &PhaseFold, + window_on: PhaseWindow, + window_off: PhaseWindow, + roi: Roi, + masked: &HashSet<(u16, u16)>, +) -> Option<(f64, f64, usize, usize)> { + let cycles = fold.validation.cycle_count; + if cycles == 0 { + return None; + } + let masked_in_roi = masked.iter().filter(|(x, y)| roi.contains(*x, *y)).count(); + let valid_pixels = roi.area().saturating_sub(masked_in_roi); + if valid_pixels == 0 { + return None; + } + + let mut on_hits: HashSet<(usize, u16, u16)> = HashSet::new(); + let mut off_hits: HashSet<(usize, u16, u16)> = HashSet::new(); + for event in &fold.events { + if !roi.contains(event.x, event.y) || masked.contains(&(event.x, event.y)) { + continue; + } + match event.polarity { + Polarity::On if window_on.contains(event.phase) => { + on_hits.insert((event.cycle_index, event.x, event.y)); + } + Polarity::Off if window_off.contains(event.phase) => { + off_hits.insert((event.cycle_index, event.x, event.y)); + } + _ => {} + } + } + + let denom = valid_pixels as f64 * cycles as f64; + Some(( + on_hits.len() as f64 / denom, + off_hits.len() as f64 / denom, + cycles, + valid_pixels, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::phase::fold_events_free_running; + use crate::types::CameraEvent; + + fn event(timestamp_us: u64, x: u16, y: u16, polarity: Polarity) -> CameraEvent { + CameraEvent { + timestamp_us, + x, + y, + polarity, + } + } + + /// Builds a fold: every pixel in an N-wide ROI fires ON near phase 0.2 and + /// OFF near phase 0.7, for `cycles` cycles at period 1000 us. + fn respond_fold(cycles: u64, pixels: u16) -> PhaseFold { + let mut events = Vec::new(); + for cycle in 0..cycles { + let base = cycle * 1_000; + for x in 0..pixels { + events.push(event(base + 200, x, 0, Polarity::On)); + events.push(event(base + 700, x, 0, Polarity::Off)); + } + } + // One trailing event so the free-running fold spans `cycles` whole cycles. + events.push(event(cycles * 1_000 + 10, 0, 0, Polarity::On)); + fold_events_free_running(&events, 1_000.0).expect("whole cycles") + } + + fn windows_disjoint(on: &PhaseWindow, off: &PhaseWindow) -> bool { + (0..HIST_BINS).all(|bin| { + let phase = (bin as f64 + 0.5) / HIST_BINS as f64; + !(on.contains(phase) && off.contains(phase)) + }) + } + + #[test] + fn auto_windows_are_separated_and_classify_a_full_response() { + let fold = respond_fold(20, 4); + let (on, off) = auto_windows(&fold, 0.1).expect("windows"); + assert!(windows_disjoint(&on, &off)); + assert_ne!(on, off); + // Free-running fold anchors phase 0 to the first event (an ON), so the ON + // cluster sits at phase 0.0 and the OFF cluster half a cycle later at 0.5. + assert!(on.contains(0.0) && off.contains(0.5)); + assert!(!on.contains(0.5) && !off.contains(0.0)); + + let roi = Roi { + x0: 0, + y0: 0, + x1: 4, + y1: 1, + }; + let (q_on, q_off, _, valid) = + response_probability(&fold, on, off, roi, &HashSet::new()).expect("counts"); + assert_eq!(valid, 4); + assert!(q_on > 0.98 && q_off > 0.98, "q_on={q_on} q_off={q_off}"); + } + + #[test] + fn partial_pixel_response_gives_proportional_probability() { + let roi = Roi { + x0: 0, + y0: 0, + x1: 4, + y1: 1, + }; + let (on, off) = auto_windows(&respond_fold(20, 4), 0.1).expect("windows"); + + // Only 2 of 4 ROI pixels respond every cycle -> q_on ~ 0.5. + let weak = respond_fold(20, 2); + let (q_on_weak, _, _, valid) = + response_probability(&weak, on, off, roi, &HashSet::new()).expect("counts"); + assert_eq!(valid, 4); + assert!((q_on_weak - 0.5).abs() < 0.05, "q_on_weak={q_on_weak}"); + } + + #[test] + fn masked_pixels_are_subtracted_from_valid_count() { + let roi = Roi { + x0: 0, + y0: 0, + x1: 4, + y1: 1, + }; + let mut masked = HashSet::new(); + masked.insert((3_u16, 0_u16)); + let fold = respond_fold(10, 4); + let (on, off) = auto_windows(&fold, 0.1).expect("windows"); + let (_, _, _, valid) = response_probability(&fold, on, off, roi, &masked).expect("counts"); + assert_eq!(valid, 3); + } +} diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs new file mode 100644 index 0000000..b1baedd --- /dev/null +++ b/plugins/stage-a-a1/src/runtime.rs @@ -0,0 +1,3460 @@ +//! Live A1 recording coordinator. +//! +//! A1 has two jobs on the Stage-A bench, both deliberately thin: +//! +//! 1. **Recording coordinator.** One *Start recording* button records, for a fixed +//! duration, the camera **RAW** stream (host recording) and the photodiode **PDQ** +//! stream (leased `stage-a.photodiode` service) together, grouped under a +//! per-`(I_k, f)` measurement **id** and a shared `_` file stem, and +//! writes an A1 **config sidecar** (`.toml`) linking the two files with the +//! modulation settings, the photodiode-measured modulation depth `a`, the ROI, and +//! the trigger info needed to reproduce and analyse the run offline. A1 owns no +//! hardware and, outside the leased sweep below, never drives the Teensy — the +//! optical drive is armed in the modulation plugin; A1 only *reads* its published +//! settings into the sidecar. The **amplitude sweep** (ADR 010) is the one scoped +//! exception: per sweep point it retargets the armed drive's *depth* through the +//! leased modulation service (`SetOpticalDepth`), waits for the photodiode-measured +//! `a` to settle, and records the point through the same coordinator. +//! +//! 2. **Live sanity quicklooks.** Folding the camera event stream on the modulation +//! period `T` (defined by the firmware phase-0 `EXT_TRIGGER`), it renders the +//! **rolling half-period response** `S_p(t)` (a live "are events appearing, is the +//! ON/OFF timing sane?" indicator) and the **response probability** `q_p` curve +//! (frozen-window Bernoulli statistic vs the measured `a`). The authoritative +//! `q_p(a, f)` fit is computed offline from the recordings; the live plot is a +//! quicklook. + +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use augur_plugin_api::{ + export_plugin, EventStoreHandle, FfiCdEvent, GlobalSettings, HostCommand, HostCommandOutcome, + HostCommandReply, HostCommandRequest, HostContext, HostDatasetDescriptor, HostDatasetKind, + HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, + PathDialogKind, Plugin, PluginCapabilities, PluginControlContext, PluginControlInbox, + PluginDiscontinuity, PluginFrame, PluginInput, PluginRuntimeRole, PluginServiceOutcome, + PluginServiceReply, PluginServiceRequest, RoiV1, Series1dLine, Series1dPoint, Series1dV1, + SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, TableColumn, + TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, TableValueType, + CTX_GLOBAL_SETTINGS, +}; +use serde::Serialize; +use serde_json::{json, Value}; +use stage_a_plugin_contract::{ + ClientId, ConnectionStateV1, LeaseId, ModulationCommandV1, ModulationRequestV1, + ModulationStateV1, PdqReceiptV1, PdqStartSpecV1, PhotodiodeCommandV1, PhotodiodeRequestV1, + PhotodiodeResponseV1, PhotodiodeSummaryV1, RequestId, RunId, SemanticRevision, WaveformV1, + CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + SERVICE_STAGE_A_MODULATION_CONTROL_V1, SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, +}; + +use crate::phase::{fold_events, fold_events_free_running, MarkerValidationConfig, PhaseFold}; +use crate::rates::{rolling_half_period_response, RollingResponsePoint}; +use crate::response_curve::{auto_windows, response_probability, PhaseWindow, ResponsePoint, Roi}; +use crate::types::{CameraEvent, Polarity}; + +const MODULATION_PLUGIN_ID: &str = "stage-a.modulation"; +const PHOTODIODE_PLUGIN_ID: &str = "stage-a.photodiode"; +const A1_PLUGIN_ID: &str = "stage-a.a1"; + +const STATUS_DATASET_ID: &str = "stage-a-a1.status"; +const STATUS_VIEW_ID: &str = "stage-a-a1.status.view"; +const ROLLING_DATASET_ID: &str = "stage-a-a1.rolling-response"; +const ROLLING_VIEW_ID: &str = "stage-a-a1.rolling-response.view"; +const RESPONSE_CURVE_DATASET_ID: &str = "stage-a-a1.response-curve"; +const RESPONSE_CURVE_VIEW_ID: &str = "stage-a-a1.response-curve.view"; + +/// Camera events retained for the live fold. At the bench event rates this is a +/// few seconds of history and keeps the fold cost bounded. +const MAX_EVENTS: usize = 4_000_000; +/// Sample points on the rolling half-period trace. +const ROLLING_SAMPLES: u64 = 256; +/// Default `q_p` window floor: grow each ON/OFF window until it falls to this +/// fraction of its histogram peak (or the opposite polarity takes over). +const DEFAULT_WINDOW_FLOOR: f64 = 0.10; +/// Default analysis window (ms) pulled from the retained EventStore each frame. +const DEFAULT_ANALYSIS_WINDOW_MS: i64 = 2_000; +/// Give up waiting for a control-plane reply after this many milliseconds. +const REPLY_TIMEOUT_MS: u64 = 15_000; +/// Upper bound on retained phase-0 markers in the no-EventStore fallback path. +const MAX_MARKERS: usize = 65_536; +/// Give up waiting for the photodiode-measured `a` to reach a sweep target +/// after this long and record anyway (the sidecar stores the measured value). +const SWEEP_SETTLE_TIMEOUT_MS: u64 = 30_000; + +/// Absolute/relative tolerance for "the measured `a` reached the sweep target". +fn sweep_tolerance(target_a: f64) -> f64 { + (target_a * 0.10).max(0.05) +} + +trait RecordingControl { + fn request_service(&mut self, request: &PluginServiceRequest); + fn request_host(&mut self, request: &HostCommandRequest); +} + +impl RecordingControl for PluginControlContext<'_> { + fn request_service(&mut self, request: &PluginServiceRequest) { + let _ = PluginControlContext::request_service(self, request); + } + + fn request_host(&mut self, request: &HostCommandRequest) { + let _ = PluginControlContext::request_host(self, request); + } +} + +/// Forwards momentary button presses across the host's UI-mirror → live-worker +/// settings snapshot. A click arrives as `true` on the clicked instance; the +/// other instance only ever sees the snapshot value from `get_setting`, so the +/// press is transported as a monotonic counter and a counter advance counts as +/// one press edge. The first counter a fresh instance sees is adopted silently +/// so a reloaded worker does not replay old presses. +#[derive(Debug, Default, Clone, Copy)] +struct PressLatch { + counter: u64, + seen: Option, +} + +impl PressLatch { + /// Interprets a settings write to this button; returns true on a press edge. + fn accept(&mut self, value: &Value) -> bool { + if value.as_bool() == Some(true) { + self.counter += 1; + self.seen = Some(self.counter); + return true; + } + let Some(incoming) = value.as_u64() else { + return false; + }; + match self.seen { + None => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + false + } + Some(seen) if incoming > seen => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + true + } + Some(_) => false, + } + } + + fn value(&self) -> Value { + json!(self.counter) + } +} + +/// Where the coordinated recording is in its lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RecPhase { + Idle, + /// Camera start sent; waiting until the host has switched into recording. + StartingCamera, + /// Camera is running; reconnecting the photodiode after the pipeline switch. + ConnectingPhotodiode, + /// AcquireLease sent to the photodiode; waiting for the grant. + AcquiringLease, + /// Camera is running; waiting for the photodiode PDQ start receipt. + StartingPhotodiode, + /// Camera RAW + photodiode PDQ recording are both in flight. + Running, + /// Photodiode finalize sent; camera keeps recording until PDQ is closed. + StoppingPhotodiode, + /// PDQ is closed; waiting for the host camera finalize receipt. + StoppingCamera, +} + +/// What a recording is for within one `(I_k, f)` measurement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RecRole { + /// One amplitude point of the sweep. + Normal, + /// Bright reference that freezes the ON/OFF windows for the whole row. + Pilot, + /// Unmodulated (`a≈0`) reference that gives the false-response floor. + Background, +} + +impl RecRole { + /// Filename-stem suffix, empty for a normal sweep point. + fn suffix(self) -> &'static str { + match self { + RecRole::Normal => "", + RecRole::Pilot => "_pilot", + RecRole::Background => "_background", + } + } + + fn label(self) -> &'static str { + match self { + RecRole::Normal => "point", + RecRole::Pilot => "pilot", + RecRole::Background => "background", + } + } +} + +/// One coordinated `(camera RAW + photodiode PDQ + sidecar)` recording. +struct Recording { + phase: RecPhase, + role: RecRole, + id: String, + stem: String, + folder: String, + duration_s: u64, + start_unix_ms: u64, + last_activity_ms: u64, + lease_id: LeaseId, + stop_requested: bool, + // outstanding request-id correlation + connect_req: u64, + lease_req: u64, + cam_start_req: u64, + cam_stop_req: u64, + pd_begin_req: u64, + pd_finalize_req: u64, + // captured receipts + connect_accepted: bool, + lease_granted: bool, + cam_raw_path: Option, + cam_finalized_path: Option, + /// True only for a complete host finalization receipt, not a partial file. + cam_complete: bool, + /// The host rejected StartRecording — skip the stop and don't wait for a + /// finalize receipt. + cam_rejected: bool, + pd_pdq_path: Option, + pd_sidecar_path: Option, + pd_finalized: bool, + pd_valid: bool, + /// The photodiode rejected BeginRecording — skip the finalize and don't + /// wait for its receipt. + pd_rejected: bool, +} + +/// Where the amplitude sweep is within its per-point cycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SweepPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// SetOpticalDepth for the current point sent; waiting for Applied. + SettingDepth, + /// Waiting for the photodiode-measured `a` to settle at the target. + Settling, + /// The per-point recording coordinator owns this phase. + Recording, +} + +/// One "record every point of the amplitude range" run: per point the sweep +/// retargets the leased modulation drive, waits for the photodiode-measured +/// `a` to settle, and hands off to the normal recording coordinator. +struct Sweep { + phase: SweepPhase, + /// Requested `a` per point, ascending over `[min_a, max_a]`. + points: Vec, + index: usize, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + depth_req: u64, + depth_applied: bool, + /// Instant the measured `a` first satisfied the tolerance, for the dwell. + settled_since_ms: Option, + /// Give-up deadline for the settle phase. + settle_deadline_ms: u64, + /// Whether the current point's recording actually started (vs. was + /// refused by validation before it began). + point_started: bool, + last_activity_ms: u64, + stop_requested: bool, +} + +impl Sweep { + fn target_a(&self) -> f64 { + self.points.get(self.index).copied().unwrap_or(0.0) + } + + fn total(&self) -> usize { + self.points.len() + } +} + +pub struct StageAA1Plugin { + enabled: bool, + runtime_role: PluginRuntimeRole, + /// While true, camera events are folded into the live quicklooks. This does + /// not record anything — recording is the separate coordinator below. + live: bool, + modulation: Option, + photodiode: Option, + camera_events: Vec, + /// Reusable buffer for exact events pulled from the retained EventStore. + event_scratch: Vec, + /// Sliding analysis window (ms) for the live fold. + analysis_window_ms: i64, + /// Rising `EXT_TRIGGER` timestamps (firmware phase-0 sync). When present these + /// anchor the fold to the drive on the camera clock; empty falls back to the + /// free-running fold on `T`. + camera_markers_us: Vec, + valid_pixels: usize, + frame_width: u16, + frame_height: u16, + // -- host camera ROI/mask, mirrored from CTX_GLOBAL_SETTINGS -- + host_roi: Option, + masked_pixels: HashSet<(u16, u16)>, + // -- response curve (auto-windowed Bernoulli q_p) -- + /// Window floor as a fraction of the ON/OFF histogram peak (see `auto_windows`). + window_floor: f64, + response_points: Vec, + /// ON/OFF windows frozen from the pilot for the current measurement row. When + /// set they override the per-fold auto-windows so the row's `q_p` is + /// consistent; loaded from the pilot's sidecar in the measurement folder. + pilot_windows: Option<(PhaseWindow, PhaseWindow)>, + /// Background floor `(q0_on, q0_off)` from the `a≈0` reference. + background_floor: Option<(f64, f64)>, + // -- recording coordinator -- + output_folder: String, + measurement_id: String, + /// Sweep range `[min_a, max_a]` for this `(I_k, f)` row (automation template). + min_a: f64, + max_a: f64, + duration_s: i64, + recording: Recording, + /// Whether the most recent recording reached its finalize path (vs. being + /// aborted); the sweep uses this to decide between advancing and stopping. + recording_completed_ok: bool, + request_seq: u64, + pd_revision_seq: u64, + /// Role latched by the Start/Pilot/Background buttons, consumed next tick. + pending_role: Option, + /// `(folder, id)` last scanned for pilot/background sidecars, so the folder is + /// re-read only when the measurement changes. + loaded_key: Option<(String, String)>, + /// One-line operator feedback about the most recent recording action. + message: String, + dataset_generation: u64, + // -- amplitude sweep -- + /// Number of sweep points across `[min_a, max_a]`. + sweep_count: i64, + /// Dwell the measured `a` must hold the target tolerance before recording. + settle_s: f64, + /// Latched by the Start sweep button, consumed next control tick. + sweep_pending: bool, + sweep: Option, + // -- momentary-button press forwarding (see PressLatch) -- + press_start: PressLatch, + press_pilot: PressLatch, + press_background: PressLatch, + press_stop: PressLatch, + press_sweep: PressLatch, + press_clear: PressLatch, + press_record_point: PressLatch, + press_clear_curve: PressLatch, +} + +impl Default for StageAA1Plugin { + fn default() -> Self { + Self { + enabled: false, + runtime_role: PluginRuntimeRole::UiMirror, + live: false, + modulation: None, + photodiode: None, + camera_events: Vec::new(), + event_scratch: Vec::new(), + analysis_window_ms: DEFAULT_ANALYSIS_WINDOW_MS, + camera_markers_us: Vec::new(), + valid_pixels: 0, + frame_width: 0, + frame_height: 0, + host_roi: None, + masked_pixels: HashSet::new(), + window_floor: DEFAULT_WINDOW_FLOOR, + response_points: Vec::new(), + pilot_windows: None, + background_floor: None, + output_folder: String::new(), + measurement_id: generate_measurement_id(), + min_a: 0.0, + max_a: 2.0, + duration_s: 10, + recording: Recording::idle(), + recording_completed_ok: false, + request_seq: 0, + pd_revision_seq: 0, + pending_role: None, + loaded_key: None, + message: String::new(), + dataset_generation: 1, + sweep_count: 5, + settle_s: 2.0, + sweep_pending: false, + sweep: None, + press_start: PressLatch::default(), + press_pilot: PressLatch::default(), + press_background: PressLatch::default(), + press_stop: PressLatch::default(), + press_sweep: PressLatch::default(), + press_clear: PressLatch::default(), + press_record_point: PressLatch::default(), + press_clear_curve: PressLatch::default(), + } + } +} + +impl Recording { + fn idle() -> Self { + Self { + phase: RecPhase::Idle, + role: RecRole::Normal, + id: String::new(), + stem: String::new(), + folder: String::new(), + duration_s: 0, + start_unix_ms: 0, + last_activity_ms: 0, + lease_id: LeaseId::new(String::new()), + stop_requested: false, + connect_req: 0, + lease_req: 0, + cam_start_req: 0, + cam_stop_req: 0, + pd_begin_req: 0, + pd_finalize_req: 0, + connect_accepted: false, + lease_granted: false, + cam_raw_path: None, + cam_finalized_path: None, + cam_complete: false, + cam_rejected: false, + pd_pdq_path: None, + pd_sidecar_path: None, + pd_finalized: false, + pd_valid: false, + pd_rejected: false, + } + } + + fn is_active(&self) -> bool { + self.phase != RecPhase::Idle + } + + fn state_label(&self) -> &'static str { + match self.phase { + RecPhase::Idle => "idle", + RecPhase::StartingCamera => "starting camera", + RecPhase::ConnectingPhotodiode => "connecting photodiode", + RecPhase::AcquiringLease => "acquiring lease", + RecPhase::StartingPhotodiode => "starting photodiode", + RecPhase::Running => "recording", + RecPhase::StoppingPhotodiode => "finalizing photodiode", + RecPhase::StoppingCamera => "finalizing camera", + } + } + + /// Seconds remaining in the fixed-duration window, when running. + fn remaining_s(&self, now_ms: u64) -> Option { + if self.phase != RecPhase::Running { + return None; + } + let elapsed_ms = now_ms.saturating_sub(self.start_unix_ms); + let total_ms = self.duration_s.saturating_mul(1_000); + Some(total_ms.saturating_sub(elapsed_ms) / 1_000) + } +} + +impl StageAA1Plugin { + fn bump(&mut self) { + self.dataset_generation = self.dataset_generation.wrapping_add(1); + } + + /// Sets the concise operator-facing recording result. + fn note(&mut self, message: impl Into) { + self.message = message.into(); + self.bump(); + } + + /// The modulation period `T` in microseconds: measured from the phase-0 + /// markers when present (the trigger *defines* the frequency, latency- + /// invariant), otherwise the modulation plugin's acknowledged waveform. + fn period_us(&self) -> Option { + if let Some(period) = self.measured_period_us() { + return Some(period); + } + let hz = self.acknowledged_frequency_hz()?; + (hz > 0.0).then(|| 1_000_000.0 / hz) + } + + /// Modulation period measured from the phase-0 markers (mean spacing). + fn measured_period_us(&self) -> Option { + if self.camera_markers_us.len() < 2 { + return None; + } + let first = *self.camera_markers_us.first()?; + let last = *self.camera_markers_us.last()?; + let spans = (self.camera_markers_us.len() - 1) as f64; + let period = last.saturating_sub(first) as f64 / spans; + (period > 0.0).then_some(period) + } + + /// Frequency (Hz) from the modulation plugin's acknowledged periodic waveform. + fn acknowledged_frequency_hz(&self) -> Option { + let target = self.modulation.as_ref()?.acknowledged.as_ref()?; + match target.waveform.as_ref()? { + WaveformV1::Periodic { + frequency_millihz, .. + } => (*frequency_millihz > 0).then(|| *frequency_millihz as f64 / 1_000.0), + _ => None, + } + } + + fn is_marker_anchored(&self) -> bool { + self.camera_markers_us.len() >= 2 + } + + fn frequency_source(&self) -> &'static str { + if self.measured_period_us().is_some() { + "trigger" + } else { + "modulation" + } + } + + fn current_fold(&self) -> Option { + let period_us = self.period_us()?; + let marker_fold = self.is_marker_anchored().then(|| { + let expected_hz = 1_000_000.0 / period_us; + fold_events( + &self.camera_events, + &self.camera_markers_us, + MarkerValidationConfig { + expected_frequency_hz: expected_hz, + // Live quicklook: accept real-world drift/jitter rather than + // rejecting the whole fold. + frequency_tolerance_fraction: 0.5, + max_period_jitter_fraction: 0.75, + expected_cycles: None, + }, + ) + .ok() + }); + // A marker glitch (dropped trigger, out-of-tolerance jitter) must not + // blank the live plots — fall back to the free-running fold on T. + marker_fold + .flatten() + .or_else(|| fold_events_free_running(&self.camera_events, period_us)) + } + + /// Optical modulation depth `a` published by the photodiode plugin. + fn measured_a(&self) -> Option { + self.photodiode + .as_ref()? + .optical_summary + .as_ref() + .map(|summary| summary.measured_log_contrast) + } + + /// Current ROI from the host camera config, clamped to the frame. + fn roi(&self) -> Option { + if self.frame_width == 0 || self.frame_height == 0 { + return None; + } + let host = self.host_roi.unwrap_or_default(); + let x0 = host.x.min(self.frame_width); + let y0 = host.y.min(self.frame_height); + let x1 = if host.width == 0 { + self.frame_width + } else { + host.x.saturating_add(host.width).min(self.frame_width) + }; + let y1 = if host.height == 0 { + self.frame_height + } else { + host.y.saturating_add(host.height).min(self.frame_height) + }; + (x1 > x0 && y1 > y0).then_some(Roi { x0, y0, x1, y1 }) + } + + /// Number of valid pixels: ROI area minus masked pixels inside it. + fn valid_pixel_count(&self) -> Option { + let roi = self.roi()?; + let masked = self + .masked_pixels + .iter() + .filter(|(x, y)| roi.contains(*x, *y)) + .count(); + Some(roi.area().saturating_sub(masked)) + } + + /// ON/OFF phase windows for `q_p`: the pilot-frozen windows when a pilot has + /// been recorded for this row, otherwise the per-fold auto-windows. + fn current_windows(&self) -> Option<(PhaseWindow, PhaseWindow)> { + if let Some(windows) = self.pilot_windows { + return Some(windows); + } + auto_windows(&self.current_fold()?, self.window_floor) + } + + /// Whether the `q_p` windows are frozen from a pilot (vs live auto-windows). + fn windows_are_frozen(&self) -> bool { + self.pilot_windows.is_some() + } + + /// ON/OFF response probability for the current fold against `current_windows`. + fn current_response(&self) -> Option<(f64, f64, usize, usize)> { + let fold = self.current_fold()?; + let roi = self.roi()?; + let (window_on, window_off) = self.current_windows()?; + response_probability(&fold, window_on, window_off, roi, &self.masked_pixels) + } + + /// Freezes the ON/OFF windows for this row from the current fold (a pilot). + fn freeze_pilot_windows(&mut self) { + match self + .current_fold() + .and_then(|fold| auto_windows(&fold, self.window_floor)) + { + Some(windows) => { + self.pilot_windows = Some(windows); + self.note("Pilot windows frozen from the live signal"); + } + None => { + self.note("No live signal to freeze windows — enable Live analysis first"); + } + } + } + + /// Captures the background floor `(q0_on, q0_off)` from the current fold. + fn capture_background_floor(&mut self) { + match self.current_response() { + Some((q_on, q_off, _, _)) => { + self.background_floor = Some((q_on, q_off)); + self.note(format!("Background floor captured (q0_on={q_on:.3})")); + } + None => { + self.note("No valid background window yet (need events and a valid ROI)"); + } + } + } + + /// Records one response-curve point at the current photodiode-measured `a`. + fn record_response_point(&mut self) -> Result<(), String> { + let measured_a = self + .measured_a() + .ok_or("no photodiode-measured a available (connect the photodiode)")?; + let (q_on, q_off, cycles, valid_pixels) = self + .current_response() + .ok_or("no valid response window yet (need trigger-anchored events and a valid ROI)")?; + self.response_points.push(ResponsePoint { + measured_a, + q_on, + q_off, + cycles, + valid_pixels, + }); + Ok(()) + } + + fn response_curve_dataset(&self) -> Series1dV1 { + let line = |select: fn(&ResponsePoint) -> f64| { + let mut points: Vec = self + .response_points + .iter() + .map(|point| Series1dPoint { + x: point.measured_a, + y: select(point), + }) + .collect(); + points.sort_by(|a, b| a.x.total_cmp(&b.x)); + points + }; + Series1dV1 { + x_label: "Measured modulation depth a = ln(I_max / I_min)".into(), + y_label: "Response probability q_p = fraction of pixel-cycles that fired".into(), + lines: vec![ + Series1dLine { + name: "ON".into(), + points: line(|point| point.q_on), + }, + Series1dLine { + name: "OFF".into(), + points: line(|point| point.q_off), + }, + ], + } + } + + fn rolling_dataset(&self) -> Series1dV1 { + const X: &str = "Camera time since first event (s)"; + const Y: &str = "Events per valid pixel in the trailing half-cycle T/2"; + let empty = || Series1dV1 { + x_label: X.into(), + y_label: Y.into(), + lines: vec![ + Series1dLine { + name: "ON".into(), + points: Vec::new(), + }, + Series1dLine { + name: "OFF".into(), + points: Vec::new(), + }, + ], + }; + let Some(fold) = self.current_fold() else { + return empty(); + }; + let first = fold.validation.first_marker_us; + let last = fold.validation.last_marker_us; + let samples = ROLLING_SAMPLES.min(last.saturating_sub(first).saturating_add(1)); + if samples < 2 { + return empty(); + } + let sample_times: Vec = (0..samples) + .map(|index| first + (last - first) * index / (samples - 1)) + .collect(); + let line = |polarity: Polarity| { + rolling_half_period_response(&fold, polarity, self.valid_pixels, &sample_times, None) + .map(|points| points_for(&points, first)) + .unwrap_or_default() + }; + Series1dV1 { + x_label: X.into(), + y_label: Y.into(), + lines: vec![ + Series1dLine { + name: "ON".into(), + points: line(Polarity::On), + }, + Series1dLine { + name: "OFF".into(), + points: line(Polarity::Off), + }, + ], + } + } + + /// Latest rolling half-period value per polarity, for the status readout. + fn latest_rolling(&self) -> Option<(f64, f64)> { + let fold = self.current_fold()?; + let at = [fold.validation.last_marker_us]; + let value = |polarity| { + rolling_half_period_response(&fold, polarity, self.valid_pixels, &at, None) + .ok() + .and_then(|points| points.first().map(|point| point.run_per_pixel)) + }; + Some((value(Polarity::On)?, value(Polarity::Off)?)) + } + + fn status_dataset(&self) -> TableDatasetV1 { + let now_ms = now_unix_ms(); + let period_us = self.period_us(); + let frequency = period_us.map(|t| 1_000_000.0 / t); + let source = self.frequency_source(); + let (on_now, off_now) = self + .latest_rolling() + .map_or((None, None), |(on, off)| (Some(on), Some(off))); + let cell = |id: &str, value: String| TableColumnData { + column_id: id.into(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + cell("state", self.recording.state_label().into()), + cell( + "measurement_id", + if self.recording.is_active() { + self.recording.id.clone() + } else { + self.measurement_id.clone() + }, + ), + cell( + "remaining", + self.recording + .remaining_s(now_ms) + .map_or_else(|| "—".into(), |s| format!("{s} s")), + ), + cell( + "frequency", + frequency.map_or_else(|| "—".into(), |hz| format!("{hz:.3} Hz ({source})")), + ), + cell( + "a", + self.measured_a() + .map_or_else(|| "—".into(), |a| format!("{a:.3}")), + ), + cell( + "s_on", + on_now.map_or_else(|| "—".into(), |v| format!("{v:.4}")), + ), + cell( + "s_off", + off_now.map_or_else(|| "—".into(), |v| format!("{v:.4}")), + ), + cell("events", self.camera_events.len().to_string()), + cell( + "message", + if self.message.is_empty() { + "—".into() + } else { + self.message.clone() + }, + ), + ], + } + } + + fn update_snapshots(&mut self, inbox: &PluginControlInbox) { + for snapshot in &inbox.snapshots { + match (snapshot.plugin_id.as_str(), snapshot.topic.as_str()) { + (MODULATION_PLUGIN_ID, CTX_STAGE_A_MODULATION_STATE_V1) => { + if let Ok(state) = serde_json::from_value(snapshot.payload.clone()) { + self.modulation = Some(state); + } + } + (PHOTODIODE_PLUGIN_ID, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1) => { + if let Ok(summary) = serde_json::from_value(snapshot.payload.clone()) { + self.photodiode = Some(summary); + } + } + _ => {} + } + } + } + + fn next_request_id(&mut self) -> u64 { + self.request_seq = self.request_seq.wrapping_add(1); + self.request_seq + } + + /// Wraps a photodiode command in the routed service request A1 emits. + fn photodiode_request(&mut self, command: PhotodiodeCommandV1) -> PluginServiceRequest { + let request_id = self.next_request_id(); + let needs_revision = matches!( + command, + PhotodiodeCommandV1::BeginRecording { .. } + | PhotodiodeCommandV1::FinalizeRecording { .. } + | PhotodiodeCommandV1::AbortRecording { .. } + ); + let mut envelope = + PhotodiodeRequestV1::new(RequestId(request_id), ClientId::new(A1_PLUGIN_ID), command); + envelope.lease_id = Some(self.recording.lease_id.clone()); + if !self.recording.stem.is_empty() { + envelope.run_id = Some(RunId::new(self.recording.stem.clone())); + } + if needs_revision { + let observed = self + .photodiode + .as_ref() + .map(|summary| { + summary + .requested_revision + .into_iter() + .chain(summary.acknowledged_revision) + .map(|revision| revision.0) + .max() + .unwrap_or(0) + }) + .unwrap_or(0); + self.pd_revision_seq = self + .pd_revision_seq + .saturating_add(1) + .max(observed.saturating_add(1)); + envelope.requested_revision = Some(SemanticRevision(self.pd_revision_seq)); + } + envelope.target_owner_instance = self + .photodiode + .as_ref() + .map(|summary| summary.owner_instance.clone()); + envelope.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + payload: serde_json::to_value(&envelope).unwrap_or(Value::Null), + } + } + + /// String metadata embedded in both recorders' own sidecars. + fn recording_metadata(&self) -> BTreeMap { + let mut meta = BTreeMap::new(); + meta.insert("a1_measurement_id".into(), self.recording.id.clone()); + meta.insert("a1_stem".into(), self.recording.stem.clone()); + meta.insert("a1_role".into(), self.recording.role.label().into()); + meta.insert( + "a1_duration_s".into(), + self.recording.duration_s.to_string(), + ); + meta.insert("sweep_min_a".into(), format!("{:.6}", self.min_a)); + meta.insert("sweep_max_a".into(), format!("{:.6}", self.max_a)); + if let Some(sweep) = self + .sweep + .as_ref() + .filter(|sweep| sweep.phase == SweepPhase::Recording) + { + meta.insert( + "sweep_requested_a".into(), + format!("{:.6}", sweep.target_a()), + ); + meta.insert("sweep_point_index".into(), (sweep.index + 1).to_string()); + meta.insert("sweep_point_total".into(), sweep.total().to_string()); + } + if let Some(a) = self.measured_a() { + meta.insert("measured_a".into(), format!("{a:.6}")); + } + if let Some(hz) = self.period_us().map(|t| 1_000_000.0 / t) { + meta.insert("modulation_frequency_hz".into(), format!("{hz:.6}")); + } + if let Some(config) = self + .modulation + .as_ref() + .and_then(|s| s.acknowledged.as_ref()) + .and_then(|t| t.a1_configuration.as_ref()) + { + meta.insert("center_dac".into(), config.center_dac.to_string()); + meta.insert("amplitude_dac".into(), config.amplitude_dac.to_string()); + } + if let Some(n) = self.valid_pixel_count() { + meta.insert("n_valid".into(), n.to_string()); + } + meta + } + + /// Kick off a coordinated recording by starting the camera first. Called + /// on the control tick after a record button is pressed. + fn begin_recording(&mut self, context: &mut impl RecordingControl, role: RecRole) { + if self.recording.is_active() { + return; + } + if self.output_folder.trim().is_empty() { + self.note("Set an output folder before recording"); + return; + } + if self.measurement_id.trim().is_empty() { + self.note("Set a measurement id before recording"); + return; + } + let now_ms = now_unix_ms(); + let id = sanitize_stem(self.measurement_id.trim()); + // Sweep points get a stable per-point tag so the row's files sort by + // sweep order as well as by timestamp. + let sweep_tag = self + .sweep + .as_ref() + .filter(|sweep| sweep.phase == SweepPhase::Recording) + .map(|sweep| format!("_p{:02}", sweep.index + 1)) + .unwrap_or_default(); + let stem = format!( + "{id}_{}{}{sweep_tag}", + format_compact_utc(now_ms / 1_000), + role.suffix() + ); + let lease_id = LeaseId::new(format!("a1-{stem}")); + self.recording_completed_ok = false; + let mut recording = Recording::idle(); + recording.role = role; + recording.id = id; + recording.stem = stem; + recording.folder = self.output_folder.trim().to_string(); + recording.duration_s = self.duration_s.max(1) as u64; + // The measurement clock starts only after both recorders acknowledge + // that they are running. + recording.start_unix_ms = 0; + recording.last_activity_ms = now_ms; + recording.lease_id = lease_id; + self.recording = recording; + + // Capture the science reference now (after any folder scan this tick), + // from the live signal at the current drive amplitude. + match role { + RecRole::Pilot => self.freeze_pilot_windows(), + RecRole::Background => self.capture_background_floor(), + RecRole::Normal => {} + } + + self.start_camera(context); + } + + /// Re-reads pilot/background sidecars from the measurement folder when the + /// measurement (folder + id) changes, so the `q_p` plot reuses them. + fn scan_measurement_folder(&mut self) { + let folder = self.output_folder.trim().to_string(); + let id = sanitize_stem(self.measurement_id.trim()); + let key = (folder.clone(), id.clone()); + if self.loaded_key.as_ref() == Some(&key) { + return; + } + self.loaded_key = Some(key); + self.pilot_windows = None; + self.background_floor = None; + if folder.is_empty() || id.is_empty() { + return; + } + let measurement_dir = Path::new(&folder).join(&id); + let measurement_dir = measurement_dir.to_string_lossy(); + if let Some((on, off)) = load_row_windows(&measurement_dir, &id, "_pilot") { + self.pilot_windows = Some((on, off)); + } + self.background_floor = load_row_background(&measurement_dir, &id, "_background"); + } + + /// Start the host camera recorder first. Starting it switches the host from + /// preview into recording and briefly revokes plugin effects, so the PDQ + /// stream must not be opened until the host acknowledges this transition. + fn start_camera(&mut self, context: &mut impl RecordingControl) { + let subdir = self.recording.id.clone(); + let stem = self.recording.stem.clone(); + let metadata = self.recording_metadata(); + + let cam_req = self.next_request_id(); + context.request_host(&HostCommandRequest { + request_id: cam_req, + command: HostCommand::StartRecording { + run_id: stem.clone(), + base_path: format!("{subdir}/{stem}.raw"), + metadata, + }, + }); + self.recording.cam_start_req = cam_req; + self.recording.phase = RecPhase::StartingCamera; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!("Recording {}: starting camera…", self.recording.id)); + } + + fn connect_photodiode(&mut self, context: &mut impl RecordingControl) { + let request = self.photodiode_request(PhotodiodeCommandV1::Connect); + self.recording.connect_req = request.request_id; + context.request_service(&request); + self.recording.phase = RecPhase::ConnectingPhotodiode; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: connecting photodiode…", + self.recording.id + )); + } + + fn acquire_photodiode(&mut self, context: &mut impl RecordingControl) { + let ttl_ms = self + .recording + .duration_s + .saturating_mul(1_000) + .saturating_add(60_000); + let request = self.photodiode_request(PhotodiodeCommandV1::AcquireLease { ttl_ms }); + self.recording.lease_req = request.request_id; + context.request_service(&request); + self.recording.phase = RecPhase::AcquiringLease; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: preparing photodiode…", + self.recording.id + )); + } + + /// Start the PDQ only after the camera recorder is running. + fn start_photodiode(&mut self, context: &mut impl RecordingControl) { + let subdir = self.recording.id.clone(); + let stem = self.recording.stem.clone(); + let spec = PdqStartSpecV1 { + pdq_path: format!("{subdir}/{stem}_pd.pdq"), + sidecar_path: format!("{subdir}/{stem}_pd.json"), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: self.recording_metadata(), + }; + let pd_request = self.photodiode_request(PhotodiodeCommandV1::BeginRecording { + specification: spec, + }); + self.recording.pd_begin_req = pd_request.request_id; + context.request_service(&pd_request); + self.recording.phase = RecPhase::StartingPhotodiode; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: starting photodiode…", + self.recording.id + )); + } + + /// Atomically close the PDQ and release its lease while the camera + /// pipeline is still live. + fn stop_photodiode(&mut self, context: &mut impl RecordingControl) { + if self.recording.lease_granted { + let pd_request = self.photodiode_request(PhotodiodeCommandV1::ReleaseLease { + finalize_recording: true, + reason: "a1 recording complete".into(), + }); + self.recording.pd_finalize_req = pd_request.request_id; + context.request_service(&pd_request); + self.recording.phase = RecPhase::StoppingPhotodiode; + } else { + self.stop_camera(context); + return; + } + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: saving photodiode data…", + self.recording.id + )); + } + + /// Stop the host recorder after the PDQ has been safely finalized. + fn stop_camera(&mut self, context: &mut impl RecordingControl) { + if self.recording.cam_raw_path.is_some() && !self.recording.cam_rejected { + let cam_req = self.next_request_id(); + context.request_host(&HostCommandRequest { + request_id: cam_req, + command: HostCommand::StopRecording, + }); + self.recording.cam_stop_req = cam_req; + self.recording.phase = RecPhase::StoppingCamera; + self.recording.last_activity_ms = now_unix_ms(); + self.note(format!( + "Recording {}: saving camera data…", + self.recording.id + )); + } else { + self.finish_recording(context); + } + } + + fn finish_recording(&mut self, context: &mut impl RecordingControl) { + let clean = self.recording.cam_complete + && self.recording.pd_finalized + && self.recording.pd_valid + && self.recording.pd_pdq_path.is_some() + && self.recording.pd_sidecar_path.is_some(); + let sidecar = self.write_sidecar(); + let message = match (sidecar, clean) { + (Ok(path), true) => format!("Saved recording {} → {path}", self.recording.id), + (Ok(path), false) => format!( + "Recording {} was incomplete — metadata saved to {path}", + self.recording.id + ), + (Err(err), _) => format!( + "Recording {} finished, metadata save failed: {err}", + self.recording.id + ), + }; + self.recording_completed_ok = clean; + self.release_and_idle(context, message); + } + + /// Release the photodiode lease (only if we actually hold it) and return to idle. + fn release_and_idle(&mut self, context: &mut impl RecordingControl, message: String) { + if self.recording.lease_granted { + let request = self.photodiode_request(PhotodiodeCommandV1::ReleaseLease { + finalize_recording: true, + reason: "a1 recording complete".into(), + }); + context.request_service(&request); + } + self.recording = Recording::idle(); + self.note(message); + } + + /// Wraps a modulation command in the routed service request A1 emits. + fn modulation_request( + &mut self, + command: ModulationCommandV1, + lease_id: &LeaseId, + ) -> PluginServiceRequest { + let request_id = self.next_request_id(); + let mut envelope = + ModulationRequestV1::new(RequestId(request_id), ClientId::new(A1_PLUGIN_ID), command); + envelope.lease_id = Some(lease_id.clone()); + envelope.target_owner_instance = self + .modulation + .as_ref() + .map(|state| state.owner_instance.clone()); + envelope.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: MODULATION_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + payload: serde_json::to_value(&envelope).unwrap_or(Value::Null), + } + } + + fn modulation_connected(&self) -> bool { + matches!( + self.modulation.as_ref().map(|state| &state.connection), + Some(ConnectionStateV1::Connected { .. }) + ) + } + + /// The requested `a` per sweep point, ascending and inclusive of both ends. + fn sweep_points(&self) -> Vec { + let count = self.sweep_count.clamp(2, 64) as usize; + let span = self.max_a - self.min_a; + (0..count) + .map(|index| self.min_a + span * index as f64 / (count - 1) as f64) + .collect() + } + + /// Worst-case sweep duration, used as the modulation lease TTL. + fn sweep_lease_ttl_ms(&self, remaining_points: usize) -> u64 { + let per_point_ms = (self.duration_s.max(1) as u64) + .saturating_mul(1_000) + .saturating_add(SWEEP_SETTLE_TIMEOUT_MS) + .saturating_add(30_000); + (remaining_points as u64) + .saturating_mul(per_point_ms) + .saturating_add(60_000) + } + + /// Kick off the amplitude sweep: validate, then lease the modulation owner. + fn begin_sweep(&mut self, context: &mut PluginControlContext<'_>) { + if self.recording.is_active() || self.sweep.is_some() { + self.message = "A recording or sweep is already running".into(); + return; + } + if self.output_folder.trim().is_empty() { + self.message = "Set an output folder before sweeping".into(); + return; + } + if self.measurement_id.trim().is_empty() { + self.message = "Set a measurement id before sweeping".into(); + return; + } + if !self.modulation_connected() { + self.message = "Modulation owner is not connected — cannot sweep".into(); + return; + } + if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { + self.message = + "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); + return; + } + if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { + self.message = "Sweep needs max a > min a".into(); + return; + } + let points = self.sweep_points(); + let now_ms = now_unix_ms(); + let lease_id = LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))); + let ttl_ms = self.sweep_lease_ttl_ms(points.len()); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + let _ = context.request_service(&request); + let total = points.len(); + self.sweep = Some(Sweep { + phase: SweepPhase::AcquiringLease, + points, + index: 0, + lease_id, + lease_granted: false, + lease_req, + depth_req: 0, + depth_applied: false, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: false, + last_activity_ms: now_ms, + stop_requested: false, + }); + self.message = format!("Sweep: acquiring modulation lease for {total} points…"); + } + + /// Release the modulation lease (if held) and clear the sweep. + fn finish_sweep(&mut self, context: &mut PluginControlContext<'_>, message: String) { + if let Some(sweep) = self.sweep.take() { + if sweep.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 sweep finished".into(), + }, + &sweep.lease_id, + ); + let _ = context.request_service(&request); + } + } + self.message = message; + } + + /// Renew the modulation lease and retarget the drive at the current point. + fn send_sweep_depth(&mut self, context: &mut PluginControlContext<'_>) { + let Some(sweep) = self.sweep.as_ref() else { + return; + }; + let lease_id = sweep.lease_id.clone(); + let remaining = sweep.total().saturating_sub(sweep.index); + let target_a = sweep.target_a(); + let index = sweep.index; + let total = sweep.total(); + + let ttl_ms = self.sweep_lease_ttl_ms(remaining); + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + let _ = context.request_service(&renew); + + let depth = self.modulation_request( + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: (target_a * 1_000.0).round().clamp(0.0, u32::MAX as f64) as u32, + }, + &lease_id, + ); + let depth_req = depth.request_id; + let _ = context.request_service(&depth); + + let now_ms = now_unix_ms(); + if let Some(sweep) = self.sweep.as_mut() { + sweep.phase = SweepPhase::SettingDepth; + sweep.depth_req = depth_req; + sweep.depth_applied = false; + sweep.settled_since_ms = None; + sweep.point_started = false; + sweep.last_activity_ms = now_ms; + } + self.message = format!( + "Sweep point {}/{}: retargeting drive to a = {:.3}…", + index + 1, + total, + target_a + ); + } + + /// Advance the amplitude sweep one control tick. Runs before + /// `drive_recording`, so a point's recording starts on the same tick. + fn drive_sweep(&mut self, context: &mut PluginControlContext<'_>) { + if self.sweep.is_none() { + if std::mem::take(&mut self.sweep_pending) { + self.begin_sweep(context); + } + return; + } + self.sweep_pending = false; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, depth_applied, last_activity_ms, index, total) = { + let sweep = self.sweep.as_ref().expect("sweep checked above"); + ( + sweep.phase, + sweep.stop_requested, + sweep.lease_granted, + sweep.depth_applied, + sweep.last_activity_ms, + sweep.index, + sweep.total(), + ) + }; + if stop_requested && phase != SweepPhase::Recording { + let message = if self.message.is_empty() { + "Sweep stopped".into() + } else { + self.message.clone() + }; + self.finish_sweep(context, message); + return; + } + match phase { + SweepPhase::AcquiringLease => { + if lease_granted { + self.send_sweep_depth(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_sweep( + context, + "Sweep aborted: timed out acquiring the modulation lease".into(), + ); + } + } + SweepPhase::SettingDepth => { + if depth_applied { + let target = self + .sweep + .as_mut() + .map(|sweep| { + sweep.phase = SweepPhase::Settling; + sweep.settled_since_ms = None; + sweep.settle_deadline_ms = now_ms + SWEEP_SETTLE_TIMEOUT_MS; + sweep.target_a() + }) + .unwrap_or_default(); + self.message = format!( + "Sweep point {}/{}: waiting for a to settle at {target:.3}…", + index + 1, + total, + ); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_sweep( + context, + "Sweep aborted: timed out retargeting the modulation drive".into(), + ); + } + } + SweepPhase::Settling => { + let target = self.sweep.as_ref().map(Sweep::target_a).unwrap_or_default(); + let settled = self + .measured_a() + .is_some_and(|measured| (measured - target).abs() <= sweep_tolerance(target)); + let dwell_ms = (self.settle_s.max(0.0) * 1_000.0) as u64; + let mut start_recording = false; + let mut settle_timed_out = false; + if let Some(sweep) = self.sweep.as_mut() { + if settled { + let since = *sweep.settled_since_ms.get_or_insert(now_ms); + if now_ms.saturating_sub(since) >= dwell_ms { + start_recording = true; + } + } else { + sweep.settled_since_ms = None; + } + if !start_recording && now_ms >= sweep.settle_deadline_ms { + // Record anyway: the sidecar stores the *measured* a, + // so an unsettled point is still a usable sample. + start_recording = true; + settle_timed_out = true; + } + if start_recording { + sweep.phase = SweepPhase::Recording; + } + } + if start_recording { + self.pending_role = Some(RecRole::Normal); + if settle_timed_out { + self.message = format!( + "Sweep point {}/{}: a did not settle at {target:.3} — recording anyway", + index + 1, + total, + ); + } + } + } + SweepPhase::Recording => { + if self.pending_role.is_some() || self.recording.is_active() { + if self.recording.is_active() { + if let Some(sweep) = self.sweep.as_mut() { + sweep.point_started = true; + } + if stop_requested { + self.recording.stop_requested = true; + } + } + return; + } + // The recording coordinator is idle again: the point either + // finished, failed, or was refused before starting. + let point_started = self.sweep.as_ref().is_some_and(|sweep| sweep.point_started); + if stop_requested { + let message = self.message.clone(); + self.finish_sweep(context, message); + } else if !point_started || !self.recording_completed_ok { + let message = format!("Sweep aborted: {}", self.message); + self.finish_sweep(context, message); + } else if index + 1 >= total { + self.finish_sweep(context, format!("Sweep complete: {total} points recorded")); + } else { + if let Some(sweep) = self.sweep.as_mut() { + sweep.index += 1; + } + self.send_sweep_depth(context); + } + } + } + } + + /// Routes modulation-service replies belonging to the sweep. Returns true + /// when the reply was consumed. + fn on_sweep_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some((lease_req, depth_req)) = self + .sweep + .as_ref() + .map(|sweep| (sweep.lease_req, sweep.depth_req)) + else { + return false; + }; + let abort = |this: &mut Self, message: String| { + this.message = message; + if let Some(sweep) = this.sweep.as_mut() { + sweep.stop_requested = true; + } + }; + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.sweep.as_mut() { + sweep.lease_granted = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => { + abort( + self, + format!("Sweep aborted: modulation lease rejected: {message}"), + ); + } + } + true + } else if reply.request_id == depth_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.sweep.as_mut() { + sweep.depth_applied = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => { + abort( + self, + format!("Sweep aborted: drive retarget rejected: {message}"), + ); + } + } + true + } else { + false + } + } + + fn on_host_reply(&mut self, reply: &HostCommandReply) { + if reply.request_id == self.recording.cam_start_req { + match &reply.outcome { + HostCommandOutcome::RecordingStarted { + actual_raw_path, .. + } => { + self.recording.cam_raw_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + // Stop the rest of the recording; drive_recording resolves the + // abort from the current phase on the next tick. + self.note(format!("Camera recording rejected ({code}): {message}")); + self.recording.cam_rejected = true; + self.recording.stop_requested = true; + } + _ => {} + } + } else if reply.request_id == self.recording.cam_stop_req { + match &reply.outcome { + HostCommandOutcome::RecordingFinalized { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.cam_complete = true; + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::RecordingPartial { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + self.message = format!("Camera stop failed ({code}): {message}"); + self.recording.cam_rejected = true; + self.recording.last_activity_ms = now_unix_ms(); + } + _ => {} + } + } + } + + fn on_service_reply(&mut self, reply: &PluginServiceReply) { + if self.on_sweep_reply(reply) { + return; + } + let response = match &reply.outcome { + PluginServiceOutcome::Accepted { payload } => { + serde_json::from_value::(payload.clone()).ok() + } + PluginServiceOutcome::Rejected { code, message } => { + if reply.request_id == self.recording.connect_req + || reply.request_id == self.recording.lease_req + || reply.request_id == self.recording.pd_begin_req + { + self.note(format!("Photodiode start failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.stop_requested = true; + } else if reply.request_id == self.recording.pd_finalize_req { + self.note(format!("Photodiode save failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + None + } + }; + let Some(response) = response else { + return; + }; + if reply.request_id == self.recording.connect_req { + self.recording.connect_accepted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.lease_req { + self.recording.lease_granted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.pd_begin_req { + if let Some(PdqReceiptV1::Started(started)) = &response.receipt { + self.recording.pd_pdq_path = Some(started.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(started.sidecar_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + } else if reply.request_id == self.recording.pd_finalize_req { + self.recording.pd_finalized = true; + if let Some(PdqReceiptV1::Finalized(finalized)) = &response.receipt { + self.recording.pd_pdq_path = Some(finalized.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(finalized.sidecar_path.clone()); + self.recording.pd_valid = finalized.valid; + } + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + } + + /// Advance the recording state machine one control tick. + fn drive_recording(&mut self, context: &mut impl RecordingControl) { + let now_ms = now_unix_ms(); + match self.recording.phase { + RecPhase::Idle => { + if let Some(role) = self.pending_role.take() { + self.begin_recording(context, role); + } + } + RecPhase::StartingCamera => { + if self.recording.cam_rejected { + let message = self.message.clone(); + self.release_and_idle(context, message); + } else if self.recording.cam_raw_path.is_some() { + if self.recording.stop_requested { + self.stop_camera(context); + } else { + self.connect_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.cam_rejected = true; + self.release_and_idle(context, "Timed out starting camera recording".into()); + } + } + RecPhase::ConnectingPhotodiode => { + if self.recording.stop_requested && !self.recording.connect_accepted { + self.stop_camera(context); + } else if self.recording.connect_accepted { + if self.recording.stop_requested { + self.stop_camera(context); + } else { + self.acquire_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.note("Timed out connecting the photodiode"); + self.stop_camera(context); + } + } + RecPhase::AcquiringLease => { + if self.recording.pd_rejected { + self.stop_camera(context); + } else if self.recording.lease_granted { + if self.recording.stop_requested { + self.stop_photodiode(context); + } else { + self.start_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.note("Timed out preparing the photodiode"); + self.stop_camera(context); + } + } + RecPhase::StartingPhotodiode => { + if self.recording.pd_pdq_path.is_some() && self.recording.pd_sidecar_path.is_some() + { + self.recording.phase = RecPhase::Running; + self.recording.start_unix_ms = now_ms; + if self.recording.stop_requested { + self.stop_photodiode(context); + } else { + self.note(format!( + "Recording {} for {} s…", + self.recording.id, self.recording.duration_s + )); + } + } else if self.recording.pd_rejected + || now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.stop_photodiode(context); + } + } + RecPhase::Running => { + let elapsed_ms = now_ms.saturating_sub(self.recording.start_unix_ms); + let over = elapsed_ms >= self.recording.duration_s.saturating_mul(1_000); + if over || self.recording.stop_requested { + self.stop_photodiode(context); + } + } + RecPhase::StoppingPhotodiode => { + if self.recording.pd_finalized || self.recording.pd_rejected { + self.stop_camera(context); + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.pd_rejected = true; + self.recording.lease_granted = false; + self.note("Timed out saving photodiode data"); + self.stop_camera(context); + } + } + RecPhase::StoppingCamera => { + if self.recording.cam_finalized_path.is_some() + || self.recording.cam_rejected + || now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + if self.recording.cam_finalized_path.is_none() && !self.recording.cam_rejected { + self.recording.cam_rejected = true; + self.note("Timed out saving camera data"); + } + self.finish_recording(context); + } + } + } + } + + /// Build and write the A1 config sidecar linking the RAW + PDQ files. + fn write_sidecar(&self) -> Result { + let now_ms = now_unix_ms(); + let modulation = self + .modulation + .as_ref() + .and_then(|s| s.acknowledged.as_ref()); + let a1_config = modulation.and_then(|t| t.a1_configuration.as_ref()); + let optical = self + .photodiode + .as_ref() + .and_then(|s| s.optical_summary.as_ref()); + let roi = self.host_roi.unwrap_or_default(); + + let raw_path = self + .recording + .cam_finalized_path + .clone() + .or_else(|| self.recording.cam_raw_path.clone()); + let camera_bias_sidecar = raw_path.as_deref().and_then(sibling_toml); + + let doc = SidecarDoc { + measurement_id: self.recording.id.clone(), + file_stem: self.recording.stem.clone(), + role: self.recording.role.label().into(), + recorded_at_utc: format_iso_utc( + if self.recording.start_unix_ms == 0 { + now_ms + } else { + self.recording.start_unix_ms + } / 1_000, + ), + finalized_at_utc: format_iso_utc(now_ms / 1_000), + duration_s: self.recording.duration_s, + sweep: { + let point = self + .sweep + .as_ref() + .filter(|sweep| sweep.phase == SweepPhase::Recording); + SweepSidecar { + min_a: self.min_a, + max_a: self.max_a, + requested_a: point.map(Sweep::target_a), + point_index: point.map(|sweep| sweep.index + 1), + point_total: point.map(Sweep::total), + } + }, + pilot: (self.recording.role == RecRole::Pilot) + .then_some(self.pilot_windows) + .flatten() + .map(|(on, off)| PilotSidecar { + window_on_start: on.start, + window_on_end: on.end, + window_off_start: off.start, + window_off_end: off.end, + }), + background: (self.recording.role == RecRole::Background) + .then_some(self.background_floor) + .flatten() + .map(|(q_on, q_off)| BackgroundSidecar { q_on, q_off }), + modulation: ModulationSidecar { + frequency_hz: self.period_us().map(|t| 1_000_000.0 / t), + frequency_source: self.frequency_source().into(), + center_dac: a1_config.map(|c| c.center_dac), + amplitude_dac: a1_config.map(|c| c.amplitude_dac), + waveform: modulation + .and_then(|t| t.waveform.as_ref()) + .map(waveform_label), + }, + optical: OpticalSidecar { + measured_a: optical.map(|o| o.measured_log_contrast), + low_clip_fraction: optical.map(|o| o.low_clip_fraction), + high_clip_fraction: optical.map(|o| o.high_clip_fraction), + measured_frequency_hz: optical.and_then(|o| o.measured_frequency_hz), + }, + camera: CameraSidecar { + roi_x: roi.x, + roi_y: roi.y, + roi_width: roi.width, + roi_height: roi.height, + masked_pixels: self.masked_pixels.len(), + n_valid: self.valid_pixel_count(), + }, + trigger: TriggerSidecar { + marker_anchored: self.is_marker_anchored(), + marker_count: self.camera_markers_us.len(), + measured_period_us: self.measured_period_us(), + }, + files: FilesSidecar { + camera_raw: raw_path, + camera_config_sidecar: camera_bias_sidecar, + photodiode_pdq: self.recording.pd_pdq_path.clone(), + photodiode_sidecar: self.recording.pd_sidecar_path.clone(), + }, + }; + + let toml = toml::to_string_pretty(&doc).map_err(|err| err.to_string())?; + let dir = PathBuf::from(&self.recording.folder).join(&self.recording.id); + std::fs::create_dir_all(&dir).map_err(|err| err.to_string())?; + let path = dir.join(format!("{}_config.toml", self.recording.stem)); + std::fs::write(&path, toml).map_err(|err| err.to_string())?; + Ok(path.display().to_string()) + } +} + +// ---- sidecar document ------------------------------------------------------ + +#[derive(Serialize)] +struct SidecarDoc { + measurement_id: String, + file_stem: String, + role: String, + recorded_at_utc: String, + finalized_at_utc: String, + duration_s: u64, + sweep: SweepSidecar, + #[serde(skip_serializing_if = "Option::is_none")] + pilot: Option, + #[serde(skip_serializing_if = "Option::is_none")] + background: Option, + modulation: ModulationSidecar, + optical: OpticalSidecar, + camera: CameraSidecar, + trigger: TriggerSidecar, + files: FilesSidecar, +} + +#[derive(Serialize)] +struct SweepSidecar { + min_a: f64, + max_a: f64, + /// The `a` this sweep point asked the drive for (measured `a` is in + /// `[optical]`); absent on manual recordings. + #[serde(skip_serializing_if = "Option::is_none")] + requested_a: Option, + /// 1-based point position within the sweep; absent on manual recordings. + #[serde(skip_serializing_if = "Option::is_none")] + point_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + point_total: Option, +} + +/// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read +/// back to reuse them across the row. +#[derive(Serialize, serde::Deserialize)] +struct PilotSidecar { + window_on_start: f64, + window_on_end: f64, + window_off_start: f64, + window_off_end: f64, +} + +/// False-response floor written into a **background** recording's sidecar. +#[derive(Serialize, serde::Deserialize)] +struct BackgroundSidecar { + q_on: f64, + q_off: f64, +} + +/// Partial view of a config sidecar for reading the pilot/background sections +/// back; every other section is ignored. +#[derive(serde::Deserialize)] +struct RowSidecar { + #[serde(default)] + pilot: Option, + #[serde(default)] + background: Option, +} + +#[derive(Serialize)] +struct ModulationSidecar { + #[serde(skip_serializing_if = "Option::is_none")] + frequency_hz: Option, + frequency_source: String, + #[serde(skip_serializing_if = "Option::is_none")] + center_dac: Option, + #[serde(skip_serializing_if = "Option::is_none")] + amplitude_dac: Option, + #[serde(skip_serializing_if = "Option::is_none")] + waveform: Option, +} + +#[derive(Serialize)] +struct OpticalSidecar { + #[serde(skip_serializing_if = "Option::is_none")] + measured_a: Option, + #[serde(skip_serializing_if = "Option::is_none")] + low_clip_fraction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + high_clip_fraction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + measured_frequency_hz: Option, +} + +#[derive(Serialize)] +struct CameraSidecar { + roi_x: u16, + roi_y: u16, + roi_width: u16, + roi_height: u16, + masked_pixels: usize, + #[serde(skip_serializing_if = "Option::is_none")] + n_valid: Option, +} + +#[derive(Serialize)] +struct TriggerSidecar { + marker_anchored: bool, + marker_count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + measured_period_us: Option, +} + +#[derive(Serialize)] +struct FilesSidecar { + #[serde(skip_serializing_if = "Option::is_none")] + camera_raw: Option, + #[serde(skip_serializing_if = "Option::is_none")] + camera_config_sidecar: Option, + #[serde(skip_serializing_if = "Option::is_none")] + photodiode_pdq: Option, + #[serde(skip_serializing_if = "Option::is_none")] + photodiode_sidecar: Option, +} + +// ---- free functions -------------------------------------------------------- + +fn ffi_to_camera_event(event: &FfiCdEvent) -> CameraEvent { + CameraEvent { + x: event.x, + y: event.y, + timestamp_us: event.timestamp_us(), + polarity: if event.is_on() { + Polarity::On + } else { + Polarity::Off + }, + } +} + +fn points_for(points: &[RollingResponsePoint], first: u64) -> Vec { + points + .iter() + .map(|point| Series1dPoint { + x: point.timestamp_us.saturating_sub(first) as f64 / 1_000_000.0, + y: point.run_per_pixel, + }) + .collect() +} + +fn waveform_label(waveform: &WaveformV1) -> String { + match waveform { + WaveformV1::Off => "off".into(), + WaveformV1::Constant { level_dac } => format!("constant({level_dac})"), + WaveformV1::Periodic { + waveform, + min_dac, + max_dac, + frequency_millihz, + } => format!( + "periodic({waveform:?}, {min_dac}..{max_dac}, {:.3} Hz)", + *frequency_millihz as f64 / 1_000.0 + ), + } +} + +fn sibling_toml(raw_path: &str) -> Option { + let path = Path::new(raw_path); + let stem = path.file_stem()?.to_string_lossy(); + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + Some(parent.join(format!("{stem}.toml")).display().to_string()) +} + +/// Parses the newest config sidecar in `folder` for measurement `id` whose stem +/// carries `role_tag` (e.g. `_pilot`). Filenames embed a sortable timestamp, so +/// the lexicographically largest matching name is the most recent. +fn load_row_sidecar(folder: &str, id: &str, role_tag: &str) -> Option { + let prefix = format!("{id}_"); + let mut best: Option = None; + for entry in std::fs::read_dir(folder).ok()?.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if name.starts_with(&prefix) + && name.contains(role_tag) + && name.ends_with("_config.toml") + && best.as_ref().is_none_or(|current| name > *current) + { + best = Some(name); + } + } + let text = std::fs::read_to_string(Path::new(folder).join(best?)).ok()?; + toml::from_str::(&text).ok() +} + +fn load_row_windows(folder: &str, id: &str, role_tag: &str) -> Option<(PhaseWindow, PhaseWindow)> { + let pilot = load_row_sidecar(folder, id, role_tag)?.pilot?; + Some(( + PhaseWindow { + start: pilot.window_on_start, + end: pilot.window_on_end, + }, + PhaseWindow { + start: pilot.window_off_start, + end: pilot.window_off_end, + }, + )) +} + +fn load_row_background(folder: &str, id: &str, role_tag: &str) -> Option<(f64, f64)> { + let background = load_row_sidecar(folder, id, role_tag)?.background?; + Some((background.q_on, background.q_off)) +} + +/// Replace anything that is not `[A-Za-z0-9._-]` with `_` so ids are file-safe. +fn sanitize_stem(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for ch in input.chars() { + if ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-') { + out.push(ch); + } else if !out.ends_with('_') { + out.push('_'); + } + } + let trimmed = out.trim_matches('_').to_string(); + if trimmed.is_empty() { + "A1".into() + } else { + trimmed + } +} + +fn generate_measurement_id() -> String { + let ms = now_unix_ms(); + format!( + "A1-{}-{:04x}", + format_compact_date(ms / 1_000), + (ms & 0xffff) + ) +} + +fn now_unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Gregorian date for a count of days since the Unix epoch (Howard Hinnant's +/// civil-from-days algorithm). +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let year = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = (doy - (153 * mp + 2) / 5 + 1) as u32; + let month = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32; + (year + i64::from(month <= 2), month, day) +} + +fn ymd_hms(unix_secs: u64) -> (i64, u32, u32, u64, u64, u64) { + let days = (unix_secs / 86_400) as i64; + let sod = unix_secs % 86_400; + let (y, m, d) = civil_from_days(days); + (y, m, d, sod / 3_600, (sod % 3_600) / 60, sod % 60) +} + +fn format_compact_date(unix_secs: u64) -> String { + let (y, m, d, ..) = ymd_hms(unix_secs); + format!("{y:04}{m:02}{d:02}") +} + +fn format_compact_utc(unix_secs: u64) -> String { + let (y, m, d, hh, mm, ss) = ymd_hms(unix_secs); + format!("{y:04}{m:02}{d:02}-{hh:02}{mm:02}{ss:02}") +} + +fn format_iso_utc(unix_secs: u64) -> String { + let (y, m, d, hh, mm, ss) = ymd_hms(unix_secs); + format!("{y:04}-{m:02}-{d:02}T{hh:02}:{mm:02}:{ss:02}Z") +} + +impl Plugin for StageAA1Plugin { + fn name(&self) -> &'static str { + "Stage-A A1 Analysis" + } + + fn description(&self) -> &'static str { + "Stage-A A1 recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar, plus live rolling-response and response-probability quicklooks." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + } + + fn set_runtime_role(&mut self, role: PluginRuntimeRole) { + self.runtime_role = role; + } + + fn reset(&mut self) { + self.camera_events.clear(); + self.event_scratch.clear(); + self.camera_markers_us.clear(); + self.valid_pixels = 0; + self.response_points.clear(); + self.pilot_windows = None; + self.background_floor = None; + self.loaded_key = None; + self.bump(); + } + + fn on_discontinuity(&mut self, reason: PluginDiscontinuity) { + match reason { + // The host raises SettingsChanged on *every* settings sync of any + // plugin (including our own button presses). The fold window + // rebuilds itself each frame, and the response curve, pilot + // windows, and background floor are operator-owned science state — + // wiping them here made "Record point" appear dead. + PluginDiscontinuity::SettingsChanged => {} + PluginDiscontinuity::Seek + | PluginDiscontinuity::SourceChanged + | PluginDiscontinuity::HistoryEvicted => self.reset(), + } + } + + fn input_kind(&self) -> PluginInput { + PluginInput::RawEvents + } + + fn capabilities(&self) -> PluginCapabilities { + // Request retained event history so the analysis window comes exactly + // from the EventStore rather than best-effort preview frames. + PluginCapabilities { + retained_event_history: true, + } + } + + fn process_frame( + &mut self, + frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + event_store: &EventStoreHandle<'_>, + ) { + self.frame_width = frame.width(); + self.frame_height = frame.height(); + // ROI and masked pixels are owned by the host camera config, not the + // plugin; mirror the latest snapshot each frame. + if let Some(settings) = context + .get::(CTX_GLOBAL_SETTINGS) + .ok() + .flatten() + { + self.host_roi = Some(settings.roi); + self.masked_pixels = settings.masked_pixels.into_iter().collect(); + } + if !self.live { + return; + } + self.valid_pixels = usize::from(frame.width()) * usize::from(frame.height()); + + // Markers (phase-0 sync) only exist on the preview frame, so accumulate + // the rising EXT_TRIGGER edges here regardless of the event source. + // Preview windows overlap, so the same trigger arrives on several + // consecutive frames — dedup after every merge or the duplicate + // timestamps fail marker validation and blank the fold. + self.camera_markers_us.extend( + frame + .external_triggers() + .iter() + .filter(|trigger| trigger.is_rising()) + .map(|trigger| trigger.timestamp_us), + ); + self.camera_markers_us.sort_unstable(); + self.camera_markers_us.dedup(); + if self.camera_markers_us.len() > MAX_MARKERS { + let excess = self.camera_markers_us.len() - MAX_MARKERS; + self.camera_markers_us.drain(..excess); + } + + let window_end = frame.window_end_us(); + let window_us = (self.analysis_window_ms.max(1) as u64).saturating_mul(1_000); + if event_store.frame_count() > 0 { + // Exact path: rebuild the analysis window from the retained event + // history, immune to dropped preview frames. + let window_start = window_end + .saturating_sub(window_us) + .max(event_store.oldest_timestamp_us()); + self.event_scratch.clear(); + event_store.collect_events_in_range(window_start, window_end, &mut self.event_scratch); + self.camera_events.clear(); + self.camera_events + .extend(self.event_scratch.iter().map(ffi_to_camera_event)); + // Keep the marker set on the same window as the events. + self.camera_markers_us + .retain(|&marker| marker >= window_start); + } else if self.camera_events.len() < MAX_EVENTS { + // Fallback (no retained history available): accumulate the + // best-effort preview-frame events. + self.camera_events + .extend(frame.events().iter().map(ffi_to_camera_event)); + } + self.bump(); + } + + fn process_control(&mut self, context: &mut PluginControlContext<'_>) { + let inbox = context.inbox().clone(); + self.update_snapshots(&inbox); + for reply in &inbox.host_replies { + self.on_host_reply(reply); + } + for reply in &inbox.service_replies { + self.on_service_reply(reply); + } + // Reuse the pilot/background captured for this measurement when idle: when + // the folder or id changes, look them up in the folder. + if !self.recording.is_active() { + self.scan_measurement_folder(); + } + // The sweep runs first so a point's recording starts on the same tick. + self.drive_sweep(context); + self.drive_recording(context); + // The fold reflects live snapshots (T, a) even between frames. + self.bump(); + } + + fn settings_schema(&self) -> SettingsSchema { + // The record/sweep buttons stay disabled until the recording has a + // destination, instead of failing with a status message after a click. + let can_record = !self.output_folder.trim().is_empty(); + SettingsSchema { + sections: vec![ + SettingsSection { + label: "Recording".into(), + description: Some( + "Records the camera RAW stream and the photodiode PDQ stream together for \ + a fixed duration and writes an A1 config sidecar (.toml) linking them. \ + Files are grouped under the measurement id and share an _ \ + stem. Arm the optical drive in the modulation plugin first; A1 only reads \ + its settings — it never drives the Teensy. For everything to land in one \ + place, point the host output folder and the photodiode data folder at the \ + same experiment directory as this folder." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "output_folder".into(), + label: "Output folder".into(), + tooltip: Some( + "Directory where the A1 config sidecar is written. Also the \ + recommended shared experiment root for the RAW/PDQ files." + .into(), + ), + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.output_folder.clone(), + }, + }, + SettingItem { + key: "measurement_id".into(), + label: "Measurement id (one per I_k, f pair)".into(), + tooltip: Some( + "Groups every repeat of one illumination/frequency pair. Included \ + in every file name. Edit it freely or press New id." + .into(), + ), + kind: SettingKind::Text { + default: self.measurement_id.clone(), + }, + }, + SettingItem { + key: "new_id".into(), + label: "New id".into(), + tooltip: Some("Generate a fresh default measurement id.".into()), + kind: SettingKind::Button { enabled: true }, + }, + SettingItem { + key: "min_a".into(), + label: "Sweep min a".into(), + tooltip: Some( + "Low end of the modulation-depth sweep for this (I_k, f) row. \ + Stored in every sidecar as the automation template; A1 does not \ + drive it — you set the drive in the modulation plugin." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: self.min_a, + }, + }, + SettingItem { + key: "max_a".into(), + label: "Sweep max a".into(), + tooltip: Some( + "High end of the modulation-depth sweep for this (I_k, f) row \ + (also the natural amplitude for the pilot). Stored in every \ + sidecar; A1 does not drive it." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 10.0, + speed: 0.01, + default: self.max_a, + }, + }, + SettingItem { + key: "sweep_count".into(), + label: "Sweep points (count)".into(), + tooltip: Some( + "How many amplitudes the Start sweep button records, spaced \ + evenly from Sweep min a to Sweep max a (inclusive)." + .into(), + ), + kind: SettingKind::I64Drag { + min: 2, + max: 64, + default: self.sweep_count, + }, + }, + SettingItem { + key: "settle_s".into(), + label: "Sweep settle (s)".into(), + tooltip: Some( + "After retargeting the drive, the sweep waits until the \ + photodiode-measured a holds the target (±10 %, at least ±0.05) \ + for this long before recording. Gives up after 30 s and records \ + anyway — the sidecar stores the measured a." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: 60.0, + speed: 0.1, + default: self.settle_s, + }, + }, + SettingItem { + key: "duration_s".into(), + label: "Duration (s)".into(), + tooltip: Some( + "How long each recording runs before it auto-stops and finalizes." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 3_600, + default: self.duration_s, + }, + }, + SettingItem { + key: "start_recording".into(), + label: "Start recording (sweep point)".into(), + tooltip: Some( + "Acquire the photodiode lease, start the camera RAW + photodiode \ + PDQ recording, auto-stop after the duration, and write the \ + sidecar. Disabled until an output folder is selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "start_sweep".into(), + label: "Start sweep (record all points)".into(), + tooltip: Some( + "Sweeps the modulation depth over [Sweep min a, Sweep max a] in \ + the configured number of points: per point A1 leases the \ + modulation owner, retargets the armed calibrated drive, waits \ + for the photodiode-measured a to settle, and records one sweep \ + point (…_pNN) like the Start recording button. Requires the \ + modulation plugin to have a calibrated periodic/optical drive \ + armed and Sweep min a > 0. Disabled until an output folder is \ + selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_pilot".into(), + label: "Record pilot (freeze ON/OFF windows)".into(), + tooltip: Some( + "Records a bright reference for this row into the same folder \ + (…_pilot) and freezes the ON/OFF windows from the current live \ + signal. Set a high, non-saturating a in the modulation plugin \ + first. The frozen windows are reused for the whole row's q_p. \ + Disabled until an output folder is selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_background".into(), + label: "Record background (a≈0 floor)".into(), + tooltip: Some( + "Records an unmodulated reference (…_background) and captures the \ + false-response floor q0 in the current windows. Set a≈0 in the \ + modulation plugin first. Disabled until an output folder is \ + selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "stop_recording".into(), + label: "Stop (abort recording / sweep)".into(), + tooltip: Some( + "Stop and finalize the current recording before the duration \ + ends; during a sweep this also aborts the remaining points." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Live analysis".into(), + description: Some( + "Live sanity quicklook. Folds the camera event stream on the modulation \ + period T (defined by the firmware phase-0 EXT_TRIGGER) and renders the \ + rolling half-period response S_p(t): events per valid pixel in the \ + trailing T/2, ON and OFF separately. Use it to confirm events are \ + appearing and the ON/OFF timing looks sane before recording. Nothing is \ + recorded here." + .into(), + ), + default_open: true, + items: vec![ + SettingItem { + key: "live".into(), + label: "Live analysis".into(), + tooltip: Some( + "Fold incoming events into the live plots. Off freezes the plots \ + at their current values. This does not record anything." + .into(), + ), + kind: SettingKind::Bool { default: self.live }, + }, + SettingItem { + key: "analysis_window_ms".into(), + label: "Analysis window (ms)".into(), + tooltip: Some( + "Trailing window pulled exactly from the retained EventStore. \ + Longer windows cover more cycles; bounded by the host event-store \ + memory budget." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 120_000, + default: self.analysis_window_ms, + }, + }, + SettingItem { + key: "clear".into(), + label: "Clear captured events".into(), + tooltip: Some( + "Empties the fold buffer and resets the live plots.".into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Response probability q_p (live quicklook)".into(), + description: Some( + "Live view of the response-curve metric q_p: the fraction of valid \ + pixel-cycles that fire at least once in the ON/OFF phase window (unlike \ + S_p, each pixel-cycle counts at most once). The ON/OFF windows come from \ + the row's pilot when one has been recorded (frozen, in the Recording \ + section) and otherwise from the trigger-anchored fold automatically — each \ + window grows out from its histogram peak until events drop below the \ + window floor or the opposite polarity takes over. Press Record point at \ + each amplitude to append a q_p(a) dot at the photodiode-measured a. The \ + ROI and masked pixels come from the camera config. The authoritative fit \ + is computed offline from the recordings; this is a quicklook." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "window_floor".into(), + label: "Window floor (fraction of peak)".into(), + tooltip: Some( + "Each ON/OFF window grows out from its histogram peak until events \ + fall below this fraction of the peak (or the opposite polarity \ + takes over). 0.10 = stop at 10 % of the peak." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.02, + max: 0.5, + speed: 0.01, + default: self.window_floor, + }, + }, + SettingItem { + key: "record_point".into(), + label: "Record point (at current a)".into(), + tooltip: Some( + "Computes q_on/q_off for the current buffer against the \ + auto-detected windows and appends a point at the \ + photodiode-measured a." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + SettingItem { + key: "clear_curve".into(), + label: "Clear response curve".into(), + tooltip: Some("Drops the recorded response-curve points.".into()), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + ], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + "output_folder" => Some(json!(self.output_folder)), + "measurement_id" => Some(json!(self.measurement_id)), + "min_a" => Some(json!(self.min_a)), + "max_a" => Some(json!(self.max_a)), + "sweep_count" => Some(json!(self.sweep_count)), + "settle_s" => Some(json!(self.settle_s)), + "duration_s" => Some(json!(self.duration_s)), + "live" => Some(json!(self.live)), + "analysis_window_ms" => Some(json!(self.analysis_window_ms)), + "window_floor" => Some(json!(self.window_floor)), + // Button presses are exported as monotonic counters so the host's + // settings snapshot transports them from the UI mirror to the + // live worker (see PressLatch). + "start_recording" => Some(self.press_start.value()), + "record_pilot" => Some(self.press_pilot.value()), + "record_background" => Some(self.press_background.value()), + "stop_recording" => Some(self.press_stop.value()), + "start_sweep" => Some(self.press_sweep.value()), + "clear" => Some(self.press_clear.value()), + "record_point" => Some(self.press_record_point.value()), + "clear_curve" => Some(self.press_clear_curve.value()), + // New id regenerates the measurement id locally; the id itself is + // what synchronizes, so the press must not be forwarded (both + // instances would generate different ids). + "new_id" => Some(json!(false)), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + match key { + "output_folder" => { + self.output_folder = value + .as_str() + .ok_or("output_folder must be a string")? + .to_string(); + } + "measurement_id" => { + self.measurement_id = value + .as_str() + .ok_or("measurement_id must be a string")? + .to_string(); + } + "new_id" if value.as_bool() == Some(true) => { + self.measurement_id = generate_measurement_id(); + } + "min_a" => { + self.min_a = value + .as_f64() + .ok_or("min_a must be a number")? + .clamp(0.0, 10.0); + } + "max_a" => { + self.max_a = value + .as_f64() + .ok_or("max_a must be a number")? + .clamp(0.0, 10.0); + } + "sweep_count" => { + self.sweep_count = value + .as_i64() + .ok_or("sweep_count must be an integer")? + .clamp(2, 64); + } + "settle_s" => { + self.settle_s = value + .as_f64() + .ok_or("settle_s must be a number")? + .clamp(0.0, 60.0); + } + "duration_s" => { + self.duration_s = value + .as_i64() + .ok_or("duration_s must be an integer")? + .clamp(1, 3_600); + } + "start_recording" => { + if self.press_start.accept(&value) { + self.pending_role = Some(RecRole::Normal); + } + } + "record_pilot" => { + if self.press_pilot.accept(&value) { + self.pending_role = Some(RecRole::Pilot); + } + } + "record_background" => { + if self.press_background.accept(&value) { + self.pending_role = Some(RecRole::Background); + } + } + "start_sweep" => { + if self.press_sweep.accept(&value) { + self.sweep_pending = true; + } + } + "stop_recording" => { + if self.press_stop.accept(&value) { + if self.recording.is_active() { + self.recording.stop_requested = true; + } + if let Some(sweep) = self.sweep.as_mut() { + sweep.stop_requested = true; + self.message = "Sweep stop requested".into(); + } + self.sweep_pending = false; + } + } + "live" => { + self.live = value.as_bool().ok_or("live must be a boolean")?; + } + "analysis_window_ms" => { + self.analysis_window_ms = value + .as_i64() + .ok_or("analysis_window_ms must be an integer")? + .clamp(1, 120_000); + } + "clear" => { + if self.press_clear.accept(&value) { + self.camera_events.clear(); + self.event_scratch.clear(); + self.camera_markers_us.clear(); + self.valid_pixels = 0; + } + } + "window_floor" => { + self.window_floor = value + .as_f64() + .ok_or("window_floor must be a number")? + .clamp(0.02, 0.5); + } + "record_point" => { + if self.press_record_point.accept(&value) { + // Report failure via the status message: on the worker the + // press arrives through the settings snapshot, where a + // returned error would be silently dropped. + if let Err(error) = self.record_response_point() { + self.message = format!("Record point failed: {error}"); + } + } + } + "clear_curve" => { + if self.press_clear_curve.accept(&value) { + self.response_points.clear(); + } + } + "new_id" => return Ok(()), + _ => return Err(format!("unknown setting '{key}'")), + } + self.bump(); + Ok(()) + } + + fn status_entries(&self) -> Vec { + let mut entries = vec![StatusEntry::LabeledValue { + label: "Recording".into(), + value: self.recording.state_label().into(), + color: None, + }]; + if self.recording.is_active() { + if let Some(remaining) = self.recording.remaining_s(now_unix_ms()) { + entries.push(StatusEntry::Text(format!( + "{} — {remaining} s remaining", + self.recording.id + ))); + } + } + if let Some(sweep) = &self.sweep { + let phase = match sweep.phase { + SweepPhase::AcquiringLease => "leasing modulation", + SweepPhase::SettingDepth => "retargeting drive", + SweepPhase::Settling => "settling", + SweepPhase::Recording => "recording", + }; + entries.push(StatusEntry::Text(format!( + "Sweep: point {}/{} at a → {:.3} ({phase})", + sweep.index + 1, + sweep.total(), + sweep.target_a() + ))); + } + if !self.message.is_empty() { + entries.push(StatusEntry::Text(self.message.clone())); + } + match self.period_us() { + Some(period_us) => { + let source = self.frequency_source(); + entries.push(StatusEntry::Text(format!( + "T = {:.3} ms ({:.3} Hz, {source})", + period_us / 1_000.0, + 1_000_000.0 / period_us, + ))); + } + None => entries.push(StatusEntry::Text( + "No modulation period (connect modulation or the EXT_TRIGGER)".into(), + )), + } + let anchor = if self.is_marker_anchored() { + format!( + "{} phase-0 markers (trigger-anchored)", + self.camera_markers_us.len() + ) + } else { + "free-running (no EXT_TRIGGER)".into() + }; + entries.push(StatusEntry::Text(format!( + "{} events, {} valid pixels; {anchor}", + self.camera_events.len(), + self.valid_pixels + ))); + entries.push(StatusEntry::Text(match self.measured_a() { + Some(a) => format!("a = {a:.3} (photodiode)"), + None => { + let detail = self + .photodiode + .as_ref() + .map(|summary| connection_label(&summary.connection)) + .unwrap_or("no snapshot"); + format!("a = — (photodiode: {detail})") + } + })); + if let Some((on, off)) = self.latest_rolling() { + entries.push(StatusEntry::Text(format!( + "S_on = {on:.4}, S_off = {off:.4} (events/pixel per T/2)" + ))); + } + let source = if self.windows_are_frozen() { + "pilot-frozen" + } else { + "auto" + }; + let windows = self.current_windows().map_or_else( + || "windows —".into(), + |(on, off)| { + format!( + "windows ({source}) ON [{:.2},{:.2}) OFF [{:.2},{:.2})", + on.start, on.end, off.start, off.end + ) + }, + ); + let valid = self + .valid_pixel_count() + .map_or_else(|| "—".into(), |n| n.to_string()); + entries.push(StatusEntry::Text(format!( + "Response curve: {windows}, N_valid = {valid}, {} point(s)", + self.response_points.len() + ))); + if let Some((q0_on, q0_off)) = self.background_floor { + entries.push(StatusEntry::Text(format!( + "Background floor: q0_on = {q0_on:.3}, q0_off = {q0_off:.3}" + ))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + fn column(id: &str, title: &str) -> TableColumn { + TableColumn { + id: id.into(), + title: title.into(), + value_type: TableValueType::String, + } + } + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "A1 status".into(), + kind: HostDatasetKind::TableV1(TableSchema { + columns: vec![ + column("state", "Recording"), + column("measurement_id", "Measurement id"), + column("remaining", "Remaining"), + column("frequency", "Frequency"), + column("a", "a (photodiode)"), + column("s_on", "S_on"), + column("s_off", "S_off"), + column("events", "Events"), + column("message", "Message"), + ], + ..TableSchema::default() + }), + empty_message: "A1 idle".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: ROLLING_DATASET_ID.into(), + title: "A1 rolling response S_p(t) — live sanity check".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Enable Live analysis; waiting for events and a period".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: RESPONSE_CURVE_DATASET_ID.into(), + title: "A1 response probability q_p(a) — live quicklook".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Capture a pilot, then record points per amplitude".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "A1 status".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: ROLLING_VIEW_ID.into(), + title: "A1 rolling response S_p (ON/OFF)".into(), + dataset_id: ROLLING_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + HostViewDescriptor { + id: RESPONSE_CURVE_VIEW_ID.into(), + title: "A1 response probability q_p (ON/OFF)".into(), + dataset_id: RESPONSE_CURVE_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::LineSeriesWindow, + }, + ], + actions: Vec::new(), + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + ROLLING_DATASET_ID => serde_json::to_vec(&self.rolling_dataset()).ok(), + RESPONSE_CURVE_DATASET_ID => serde_json::to_vec(&self.response_curve_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + matches!( + dataset_id, + STATUS_DATASET_ID | ROLLING_DATASET_ID | RESPONSE_CURVE_DATASET_ID + ) + .then_some(self.dataset_generation) + .unwrap_or(0) + } +} + +fn connection_label(connection: &ConnectionStateV1) -> &'static str { + match connection { + ConnectionStateV1::Connected { .. } => "connected", + ConnectionStateV1::Connecting => "connecting", + ConnectionStateV1::Disconnected => "disconnected", + ConnectionStateV1::Faulted { .. } => "faulted", + } +} + +export_plugin!(StageAA1Plugin); + +#[cfg(test)] +mod tests { + use stage_a_plugin_contract::{ + OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, RequestOutcomeV1, + ResponseCommonV1, Sha256V1, StreamIntegrityV1, CONTRACT_VERSION_V1, + }; + + use super::*; + + #[derive(Default)] + struct ControlSink { + services: Vec, + hosts: Vec, + } + + impl RecordingControl for ControlSink { + fn request_service(&mut self, request: &PluginServiceRequest) { + self.services.push(request.clone()); + } + + fn request_host(&mut self, request: &HostCommandRequest) { + self.hosts.push(request.clone()); + } + } + + fn control_tick( + plugin: &mut StageAA1Plugin, + inbox: PluginControlInbox, + sink: &mut ControlSink, + ) { + for reply in &inbox.host_replies { + plugin.on_host_reply(reply); + } + for reply in &inbox.service_replies { + plugin.on_service_reply(reply); + } + plugin.drive_recording(sink); + } + + fn pd_reply(request_id: u64, receipt: Option) -> PluginServiceReply { + let response = PhotodiodeResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: RequestId(request_id), + owner_instance: OwnerInstanceId::new("pd-test"), + run_id: None, + requested_revision: None, + acknowledged_revision: None, + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + receipt, + }; + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).expect("response"), + }, + } + } + + fn on(timestamp_us: u64) -> CameraEvent { + CameraEvent { + timestamp_us, + x: 0, + y: 0, + polarity: Polarity::On, + } + } + + /// A plugin whose period comes from marker spacing (no fallback frequency). + fn plugin_with_markers() -> StageAA1Plugin { + StageAA1Plugin { + valid_pixels: 10, + camera_markers_us: vec![0, 1_000, 2_000, 3_000], + ..StageAA1Plugin::default() + } + } + + #[test] + fn period_comes_from_the_trigger_marker_spacing() { + let plugin = plugin_with_markers(); + let period = plugin.period_us().expect("measured period"); + assert!((period - 1_000.0).abs() < 1e-6, "period={period}"); + assert_eq!(plugin.frequency_source(), "trigger"); + } + + #[test] + fn no_markers_and_no_modulation_yields_no_period() { + let plugin = StageAA1Plugin::default(); + assert!(plugin.period_us().is_none()); + assert!(plugin.rolling_dataset().lines[0].points.is_empty()); + } + + #[test] + fn external_triggers_anchor_the_fold() { + let mut plugin = plugin_with_markers(); + for cycle in 0..3 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + assert!(plugin.is_marker_anchored()); + let fold = plugin.current_fold().expect("marker fold"); + assert_eq!(fold.validation.cycle_count, 3); + assert!((fold.events[0].phase - 0.2).abs() < 1e-9); + } + + #[test] + fn rolling_dataset_keeps_on_and_off_separate() { + let mut plugin = plugin_with_markers(); + for cycle in 0..3 { + let base = cycle * 1_000; + plugin.camera_events.push(on(base + 100)); + plugin.camera_events.push(CameraEvent { + polarity: Polarity::Off, + ..on(base + 600) + }); + } + let rolling = plugin.rolling_dataset(); + assert_eq!(rolling.lines.len(), 2); + assert_eq!(rolling.lines[0].name, "ON"); + assert!(rolling.lines[0].points.len() >= 2); + } + + #[test] + fn response_curve_auto_windows_without_a_pilot_and_refuses_without_a() { + let mut plugin = plugin_with_markers(); + plugin.frame_width = 8; + plugin.frame_height = 1; + for cycle in 0..20 { + let base = cycle * 1_000; + for x in 0..4 { + plugin.camera_events.push(CameraEvent { + timestamp_us: base + 200, + x, + y: 0, + polarity: Polarity::On, + }); + plugin.camera_events.push(CameraEvent { + timestamp_us: base + 700, + x, + y: 0, + polarity: Polarity::Off, + }); + } + } + plugin.camera_markers_us = (0..=20).map(|c| c * 1_000).collect(); + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 4, + height: 1, + }); + + // Windows come straight from the fold — no pilot capture needed. + let (q_on, q_off, _, valid) = plugin.current_response().expect("response"); + assert_eq!(valid, 4); + assert!(q_on > 0.9 && q_off > 0.9, "q_on={q_on} q_off={q_off}"); + // Recording a point is still refused without a photodiode-measured a. + assert!(plugin.measured_a().is_none()); + assert!(plugin.record_response_point().is_err()); + } + + #[test] + fn press_latch_distinguishes_clicks_baselines_and_advances() { + let mut latch = PressLatch::default(); + // Direct click on this instance: an edge, and the counter advances. + assert!(latch.accept(&json!(true))); + assert_eq!(latch.value(), json!(1)); + // `false` writes (legacy snapshots) are never edges. + assert!(!latch.accept(&json!(false))); + + // A fresh instance adopts the first forwarded counter silently… + let mut worker = PressLatch::default(); + assert!(!worker.accept(&json!(3))); + // …repeats are not edges… + assert!(!worker.accept(&json!(3))); + // …and only an advance is one press. + assert!(worker.accept(&json!(4))); + assert!(!worker.accept(&json!(4))); + } + + #[test] + fn forwarded_button_counter_latches_the_recording_role() { + let mut plugin = StageAA1Plugin::default(); + // First snapshot after (re)load: adopt the mirror's counter, no press. + plugin + .set_setting("start_recording", json!(2)) + .expect("baseline"); + assert!(plugin.pending_role.is_none()); + // The mirror's counter advances by one click → one press edge. + plugin + .set_setting("start_recording", json!(3)) + .expect("press"); + assert_eq!(plugin.pending_role, Some(RecRole::Normal)); + // Re-applying the same snapshot must not re-press. + plugin.pending_role = None; + plugin + .set_setting("start_recording", json!(3)) + .expect("repeat"); + assert!(plugin.pending_role.is_none()); + } + + #[test] + fn recording_orders_camera_then_pdq_and_saves_inside_the_measurement_folder() { + let folder = std::env::temp_dir().join(format!("a1-lifecycle-{}", now_unix_ms())); + let mut plugin = StageAA1Plugin { + output_folder: folder.display().to_string(), + measurement_id: "A1-row".into(), + duration_s: 1, + pending_role: Some(RecRole::Normal), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!(plugin.recording.phase, RecPhase::StartingCamera); + assert_eq!(sink.hosts.len(), 1); + assert!(sink.services.is_empty(), "PDQ must not start before camera"); + let cam_start_req = sink.hosts[0].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_start_req, + outcome: HostCommandOutcome::RecordingStarted { + actual_raw_path: "/camera/A1-row/run.raw".into(), + started_at: "2026-07-23T00:00:00Z".into(), + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let connect = sink.services.last().expect("connect request"); + let connect_envelope: PhotodiodeRequestV1 = + serde_json::from_value(connect.payload.clone()).expect("connect envelope"); + assert!(matches!( + connect_envelope.command, + PhotodiodeCommandV1::Connect + )); + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(connect.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let acquire = sink.services.last().expect("lease request"); + let acquire_envelope: PhotodiodeRequestV1 = + serde_json::from_value(acquire.payload.clone()).expect("lease envelope"); + assert!(matches!( + acquire_envelope.command, + PhotodiodeCommandV1::AcquireLease { .. } + )); + assert_eq!( + acquire_envelope.run_id.as_ref().map(RunId::as_str), + Some(plugin.recording.stem.as_str()) + ); + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(acquire.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let begin = sink.services.last().expect("begin request"); + let begin_envelope: PhotodiodeRequestV1 = + serde_json::from_value(begin.payload.clone()).expect("begin envelope"); + assert!(matches!( + begin_envelope.command, + PhotodiodeCommandV1::BeginRecording { .. } + )); + assert_eq!(begin_envelope.requested_revision, Some(SemanticRevision(1))); + assert_eq!(plugin.recording.start_unix_ms, 0); + + let run_id = begin_envelope.run_id.expect("run id"); + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply( + begin.request_id, + Some(PdqReceiptV1::Started(PdqStartedReceiptV1 { + run_id: run_id.clone(), + pdq_path: "/pd/A1-row/run_pd.pdq".into(), + sidecar_path: "/pd/A1-row/run_pd.json".into(), + opened_at_unix_ms: now_unix_ms(), + stream_epoch: 1, + first_sample_index: Some(0), + })), + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert_eq!(plugin.recording.phase, RecPhase::Running); + assert!(plugin.recording.start_unix_ms > 0); + + plugin.recording.start_unix_ms = now_unix_ms().saturating_sub(1_000); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let release = sink.services.last().expect("release request"); + let release_envelope: PhotodiodeRequestV1 = + serde_json::from_value(release.payload.clone()).expect("release envelope"); + assert!(matches!( + release_envelope.command, + PhotodiodeCommandV1::ReleaseLease { + finalize_recording: true, + .. + } + )); + assert_eq!(sink.hosts.len(), 1, "camera keeps running until PDQ closes"); + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply( + release.request_id, + Some(PdqReceiptV1::Finalized(PdqFinalizedReceiptV1 { + run_id, + pdq_path: "/pd/A1-row/run_pd.pdq".into(), + sidecar_path: "/pd/A1-row/run_pd.json".into(), + opened_at_unix_ms: now_unix_ms().saturating_sub(1_000), + finalized_at_unix_ms: now_unix_ms(), + file_size_bytes: 64, + sha256: Sha256V1::parse("ab".repeat(32)).expect("sha"), + frames_written: 1, + sample_frames_written: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + segment_count: 1, + integrity: StreamIntegrityV1::default(), + termination: stage_a_plugin_contract::PdqTerminationV1::OperatorStopped, + valid: true, + })), + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert_eq!(plugin.recording.phase, RecPhase::StoppingCamera); + assert_eq!(sink.hosts.len(), 2); + let cam_stop_req = sink.hosts[1].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_stop_req, + outcome: HostCommandOutcome::RecordingFinalized { + actual_raw_path: "/camera/A1-row/run.raw".into(), + size: 128, + sha256: "cd".repeat(32), + duration_us: 1_000_000, + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert_eq!(plugin.recording.phase, RecPhase::Idle); + let measurement_dir = folder.join("A1-row"); + let sidecars: Vec<_> = std::fs::read_dir(&measurement_dir) + .expect("measurement folder") + .flatten() + .map(|entry| entry.path()) + .collect(); + assert_eq!(sidecars.len(), 1); + assert!(sidecars[0] + .file_name() + .is_some_and(|name| name.to_string_lossy().ends_with("_config.toml"))); + assert!(plugin.message.starts_with("Saved recording A1-row")); + + std::fs::remove_dir_all(folder).expect("cleanup"); + } + + #[test] + fn duplicate_or_jittery_markers_still_yield_a_fold() { + // A long trigger dropout leaves a gap far beyond the jitter tolerance: + // marker validation rejects the fold, but the quicklook must fall back + // to the free-running fold instead of blanking. + let mut plugin = StageAA1Plugin { + valid_pixels: 10, + camera_markers_us: vec![0, 1_000, 2_000, 10_000], + ..StageAA1Plugin::default() + }; + for cycle in 0..10 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + let fold = plugin.current_fold().expect("fallback fold"); + assert!(fold.markers_us.is_empty(), "free-running fold expected"); + assert!(!plugin.rolling_dataset().lines[0].points.is_empty()); + } + + #[test] + fn sweep_points_span_the_range_inclusively() { + let plugin = StageAA1Plugin { + min_a: 0.5, + max_a: 2.5, + sweep_count: 5, + ..StageAA1Plugin::default() + }; + let points = plugin.sweep_points(); + assert_eq!(points.len(), 5); + assert!((points[0] - 0.5).abs() < 1e-12); + assert!((points[4] - 2.5).abs() < 1e-12); + assert!((points[2] - 1.5).abs() < 1e-12); + } + + #[test] + fn sweep_point_recordings_carry_the_requested_a_in_the_sidecar() { + let mut plugin = plugin_with_markers(); + plugin.min_a = 0.5; + plugin.max_a = 1.5; + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + points: vec![0.5, 1.0, 1.5], + index: 1, + lease_id: LeaseId::new("a1-sweep-test"), + lease_granted: true, + lease_req: 0, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + last_activity_ms: 0, + stop_requested: false, + }); + plugin.recording.id = "A1-sweeprow".into(); + plugin.recording.stem = "A1-sweeprow_20260723-000000_p02".into(); + plugin.recording.folder = std::env::temp_dir().display().to_string(); + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_774_224_000_000; + let path = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&path).expect("read sidecar"); + assert!(text.contains("requested_a = 1.0"), "sidecar: {text}"); + assert!(text.contains("point_index = 2")); + assert!(text.contains("point_total = 3")); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn settings_discontinuities_keep_the_response_curve() { + let mut plugin = StageAA1Plugin::default(); + plugin.response_points.push(ResponsePoint { + measured_a: 1.0, + q_on: 0.5, + q_off: 0.1, + cycles: 10, + valid_pixels: 4, + }); + plugin.on_discontinuity(PluginDiscontinuity::SettingsChanged); + assert_eq!(plugin.response_points.len(), 1); + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + assert!(plugin.response_points.is_empty()); + } + + #[test] + fn measurement_id_generation_is_file_safe_and_prefixed() { + let id = generate_measurement_id(); + assert!(id.starts_with("A1-")); + assert_eq!(sanitize_stem(&id), id); + assert_eq!(sanitize_stem("I_k 3 / f=10Hz"), "I_k_3_f_10Hz"); + } + + #[test] + fn compact_utc_formats_a_known_epoch() { + // 2026-07-23T00:00:00Z = 1_784_764_800 s; +3661 s = 01:01:01. + assert_eq!(format_compact_date(1_784_764_800), "20260723"); + assert_eq!(format_iso_utc(1_784_764_800), "2026-07-23T00:00:00Z"); + assert_eq!(format_compact_utc(1_784_764_800), "20260723-000000"); + assert_eq!( + format_iso_utc(1_784_764_800 + 3_661), + "2026-07-23T01:01:01Z" + ); + } + + #[test] + fn sidecar_serializes_the_expected_sections() { + let mut plugin = plugin_with_markers(); + plugin.frame_width = 4; + plugin.frame_height = 1; + plugin.recording.id = "A1-test".into(); + plugin.recording.stem = "A1-test_20260723-000000".into(); + plugin.recording.folder = std::env::temp_dir().display().to_string(); + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_774_224_000_000; + plugin.recording.cam_finalized_path = Some("/data/A1-test/A1-test.raw".into()); + plugin.recording.pd_pdq_path = Some("/pd/A1-test/A1-test_pd.pdq".into()); + let doc = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&doc).expect("read sidecar"); + assert!(text.contains("measurement_id = \"A1-test\"")); + assert!(text.contains("[modulation]")); + assert!(text.contains("[camera]")); + assert!(text.contains("[files]")); + assert!(text.contains("camera_config_sidecar = \"/data/A1-test/A1-test.toml\"")); + let _ = std::fs::remove_file(&doc); + } + + #[test] + fn pilot_windows_round_trip_through_the_folder() { + let dir = std::env::temp_dir().join(format!("a1-pilot-{}", now_unix_ms())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let folder = dir.display().to_string(); + + let mut plugin = plugin_with_markers(); + plugin.output_folder = folder.clone(); + plugin.measurement_id = "A1-row".into(); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.10, + end: 0.30, + }, + PhaseWindow { + start: 0.55, + end: 0.80, + }, + )); + // Write a pilot sidecar for the row. + plugin.recording.role = RecRole::Pilot; + plugin.recording.id = "A1-row".into(); + plugin.recording.stem = "A1-row_20260723-000000_pilot".into(); + plugin.recording.folder = folder.clone(); + plugin.write_sidecar().expect("pilot sidecar"); + + // A fresh plugin on the same folder+id auto-loads the frozen windows. + let mut other = StageAA1Plugin { + output_folder: folder.clone(), + measurement_id: "A1-row".into(), + ..StageAA1Plugin::default() + }; + other.scan_measurement_folder(); + let (on, off) = other.pilot_windows.expect("loaded windows"); + assert!((on.start - 0.10).abs() < 1e-9 && (off.end - 0.80).abs() < 1e-9); + assert!(other.windows_are_frozen()); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/plugins/stage-a-a1/src/types.rs b/plugins/stage-a-a1/src/types.rs new file mode 100644 index 0000000..91a39cf --- /dev/null +++ b/plugins/stage-a-a1/src/types.rs @@ -0,0 +1,26 @@ +/// Event-camera polarity. A1 always analyses ON and OFF separately. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Polarity { + On, + Off, +} + +impl Polarity { + pub const ALL: [Self; 2] = [Self::On, Self::Off]; + + pub const fn as_str(self) -> &'static str { + match self { + Self::On => "on", + Self::Off => "off", + } + } +} + +/// The event fields needed by the pure A1 analysis. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CameraEvent { + pub timestamp_us: u64, + pub x: u16, + pub y: u16, + pub polarity: Polarity, +} diff --git a/plugins/stage-a-modulation/Cargo.toml b/plugins/stage-a-modulation/Cargo.toml index 983a927..57207f9 100644 --- a/plugins/stage-a-modulation/Cargo.toml +++ b/plugins/stage-a-modulation/Cargo.toml @@ -13,4 +13,5 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true stage-a-io = { path = "../../stage-a-io" } +stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } toml = "0.8" diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index b349ba6..bb29cab 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -5,14 +5,70 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** ## What it does -- **Power slider** in DAC codes (0–4095). Its upper bound is the **max limit** setting — set that - to the highest code the connected device tolerates and the slider physically cannot exceed it. -- **Mode**: `CONST` (hold the level), `SINE`, or `SQUARE` with a **frequency** (0.01–2000 Hz) and - a **min threshold** — the periodic waveforms swing between the threshold and the slider value. +- **Drive method** selects how the DAC operating band is defined: + - `MANUAL`: **Power** is the peak/operating code and **Min threshold** is the lower endpoint. + - `CALIBRATED`: `V_null`, `Vπ`, `I_k`, and optical depth `a` determine the endpoints. + Measure `V_null`/`Vπ` with the built-in transfer sweep — see [Calibration](#calibration--measuring-v_null-and-vπ). +- **Mode** independently selects the waveform that fills that band. All five modes are available + under both methods. +- **Max limit** is always visible and is the hard DAC ceiling for every drive. - Every accepted change is sent to the Teensy **immediately** (one `MOD` command); there is no Apply button. - The panel shows the modulation and live DAC code the **board reports** (from the `MOD` reply and - a 2 Hz `STATUS` poll), not just what was commanded. + a 2 Hz `STATUS` poll), plus the selected method and resolved DAC band. + +| Mode | Manual band `[min, power]` | Calibrated band from `I_k`, `a`, `V_null`, `Vπ` | +|---|---|---| +| `CONST` | hold `power` | hold the DAC code for `I_k` | +| `DAC_SINE` | DAC sine across the band | DAC sine across the band | +| `SQUARE` | DAC square across the band | DAC square across the band | +| `OPTICAL_LOG_SINE` | optical log-sine across the band | optical log-sine about `I_k` | +| `OPTICAL_LINEAR_SINE` | optical linear-sine across the band | optical linear-sine about `I_k` | + +Manual optical modes reuse the stored `V_null`/`Vπ` lobe and derive their effective `(I_k, a)` +from the slider band through the forward optical transfer. + +In calibrated `CONST`, `a` is irrelevant: the hold is +`V_null + (2Vπ/π)·asin(sqrt(I_k))`. With `V_null=1630` and `Vπ=860`, this is +2490 at `I_k=1` and 1685 at `I_k=0.01`. Periodic modes still need optical +headroom and reject impossible `I_k`/`a` combinations without changing the +displayed setting or leaving it out of sync with the board. + +## Calibration — measuring `V_null` and `Vπ` + +Do not type these in from a datasheet. Static birefringence, alignment, PBS extinction, driver +gain, temperature, and the actual electrical load all enter the realised map, so measure them: + +1. Connect the command port **and** the photodiode plugin (the sweep reads its published level; + it needs no lease and takes no recording). +2. Set **Detector port**. Stage-A watches the PBS *reject* port, where the detector is + **brightest** at `V_null` — the default. This cannot be inferred from the sweep: a bright and + a dark extremum fit the measured curve equally well, and only the optics say which one is zero + excitation. Getting it wrong puts `V_null` a quarter wave out. +3. Press **Measure transfer curve**. It steps settled `CONST` codes across `0..max limit`, up and + back down (~20 s), and fits the lobe. Your armed drive is restored afterwards, on every exit + path. +4. Read the result in the **Pockels transfer curve** view and the status line, then press + **Apply to V_null / Vπ**. Anything questionable — a high residual, dropped points, + hysteresis, clipping — appears as a `Check:` line but does not block the apply: the plot is + the arbiter, and a single stray sample can inflate the residual fivefold while leaving `Vπ` + accurate to a few codes. Wild points are dropped from the fit automatically. + +The view also works *before* any measurement: it draws the lobe your current `V_null`/`Vπ` claim, +on a normalised axis, with markers at `V_null` and `V_null + Vπ`. + +Two properties worth knowing: + +- `V_null`/`Vπ` need **no** dark measurement and **no** total-power anchor — the fitted offset and + amplitude absorb the dark level and the front-end gain. +- The detector level at the null is reported as a **lower bound** on the total-power anchor + `I_tot`, *not* as the anchor. On the reject port the residual transmitted floor is not separable + from it; freezing a real anchor needs a transmitted-port power measurement. + +Set a **Calibration folder** to archive each applied calibration (points, fit, residual, +hysteresis) and stamp `calibration_id` into the state snapshot, so recordings can cite the +inversion they used. Full detail: [feature brief](../../docs/features/stage-a-pockels-calibration.md), +[ADR 011](../../docs/adr/011-stage-a-pockels-transfer-calibration.md). ## Connecting @@ -20,8 +76,9 @@ Controls the laser modulation input (Hermit J23, `DAC1.4`) through the Teensy ** **without a running camera** (device I/O lives in a plugin-owned thread, independent of the host's frame-driven plugin passes). Connecting never changes the output; only changes made while connected are transferred. -- **Output off = power slider at 0.** The firmware output is **set-and-hold**: disconnecting, - closing the GUI, or a crash leaves the last modulation running (`stage-a-controller` ADR 002). +- The firmware output is **set-and-hold**: disconnecting, closing the GUI, or a crash leaves the + last modulation running (`stage-a-controller` ADR 002). In Manual mode, Power `0` drives `0 V`; + automated workflows use their explicit `SafeOff` command. ## Ports @@ -32,3 +89,17 @@ controller for hardware-free testing. Replaying a recording disconnects the plugin defensively; live control itself needs no capture session. + +## Workflow-owner service + +This plugin is the sole command-port owner for manual operation and automated Stage-A workflows. +The live-worker instance exposes `stage_a.modulation.control.v1` under the stable plugin ID +`stage-a.modulation`; UI-mirror and offline instances never open the port or apply hardware +effects. Automated clients acquire a renewable lease and submit semantic, idempotent commands +(`SetWaveform`, `PrepareA1`, `StartAcquisition`, `StopAcquisition`, `SafeOff`) rather than changing +UI settings or sending raw firmware strings. While leased, manual control settings are locked. + +The bounded `stage_a.modulation_state.v1` snapshot keeps requested and board-acknowledged semantic +revisions separate. Lease expiry, replay/effects revocation, or owner shutdown during an automated +run performs a best-effort controller `STOP` followed by `MOD wave=OFF` before releasing the port. +Automation specifies exact waveforms and therefore does not use the UI Drive method. diff --git a/plugins/stage-a-modulation/plugin.toml b/plugins/stage-a-modulation/plugin.toml index 39b9d48..059aa60 100644 --- a/plugins/stage-a-modulation/plugin.toml +++ b/plugins/stage-a-modulation/plugin.toml @@ -1,6 +1,7 @@ +id = "stage-a.modulation" name = "Stage-A Modulation" -version = "0.3.0" -description = "Laser modulation control: capped power slider plus constant/sine/square drive of the Teensy DAC (J23), applied immediately." +version = "0.4.0" +description = "Laser modulation control: capped power slider plus constant/sine/square/optical drive of the Teensy DAC (J23), applied immediately, with a measured Pockels transfer calibration for V_null/Vπ." domain = "stage-a" library = "augur_plugin_stage_a_modulation" phase = "frame_only" diff --git a/plugins/stage-a-modulation/src/calibration.rs b/plugins/stage-a-modulation/src/calibration.rs new file mode 100644 index 0000000..9bddcfe --- /dev/null +++ b/plugins/stage-a-modulation/src/calibration.rs @@ -0,0 +1,786 @@ +//! Measured Pockels/PBS transfer calibration: fits `V_null` and `Vπ` from a +//! sweep of settled `CONST` DAC codes against the photodiode level. +//! +//! The operator must not have to trust a nominal `Vπ` (knowledge base: +//! `methodology/pockels-waveform-linearisation.md` §4). This module turns a +//! table of `(DAC code, detector volts)` points into the lobe parameters the +//! optical inversion in [`crate::waveform`] needs. +//! +//! # Model +//! +//! ```text +//! P(c) = p0 + p1 · sin²(π (c − V_null) / (2 Vπ)) +//! ``` +//! +//! `p1` is **signed**, because the Stage-A photodiode sits behind the PBS +//! *reject* port and measures the complement `I_pd = I_tot − I_exc`, moving +//! *against* the excitation (knowledge base: `setup/optical-path.md`). +//! +//! The sign cannot be inferred from the sweep. `sin²` is symmetric about its +//! peak, so `(v, p0, p1)` and `(v + Vπ, p0 + p1, −p1)` describe the *same* +//! measured curve exactly; the data alone cannot say which extremum is the +//! excitation null. That is a physical fact about the port, not a fit +//! parameter, so [`fit_transfer`] takes the geometry as an **input** and picks +//! the matching representation. Getting it wrong would place `V_null` a +//! quarter wave off and run the drive on the inverted branch, so it is asked +//! rather than guessed. +//! +//! Two consequences worth stating, because they remove procedure rather than +//! add it: +//! +//! - **The shape is dark- and gain-immune.** `p0` absorbs the dark level and +//! any DC offset, `p1` absorbs the front-end gain. `V_null`/`Vπ` therefore +//! need neither a dark measurement nor the total-power anchor. +//! - **The absolute scale is not recoverable here.** On the reject port the +//! residual transmitted floor cannot be separated from the anchor `I_tot` +//! (knowledge base §4.4), so this module reports the detector extrema and +//! explicitly does *not* derive a maximum achievable `a` from them. +//! +//! # Fit +//! +//! Because `sin²(x) = (1 − cos 2x)/2`, the model is exactly a constant plus +//! **one sinusoid of period `2Vπ`** — and a sinusoid of known period is linear +//! in its quadrature components. So for each candidate `Vπ` the phase (hence +//! `V_null`) and both amplitudes fall out of a 3×3 linear solve, and the +//! search is one-dimensional: scan `Vπ` over every period the sweep can +//! resolve, then refine. See [`solve_harmonic`]. +//! +//! This matters beyond elegance. Seeding the period from the measured extrema +//! — the obvious approach — breaks on exactly the sweeps that matter: with a +//! real `Vπ` near 860 the DAC range holds ~2.4 lobes, so the global minimum +//! and maximum can sit whole periods apart and the seed is meaningless. + +use std::f64::consts::PI; + +use crate::waveform::{LobeInversion, DAC_FULL_SCALE}; + +/// Sweep direction, kept per point so ascending/descending repeatability can +/// be reported (knowledge base §5 acceptance test 1). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Direction { + Ascending, + Descending, +} + +impl Direction { + pub fn label(self) -> &'static str { + match self { + Self::Ascending => "up", + Self::Descending => "down", + } + } +} + +/// One settled `(DAC code, detector level)` measurement. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SweepPoint { + pub code: u16, + pub direction: Direction, + /// Raw detector level in volts, as published by the photodiode owner. + pub volts: f64, + /// Spread over the averaged window; a settle-quality witness. + pub peak_to_peak_volts: f64, + pub clipped: bool, +} + +/// Which port the detector watches. An input to the fit, not an output: the +/// swept curve is identical either way (see the module docs), so this states +/// the bench geometry that resolves the ambiguity. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DetectorGeometry { + /// Detector darkens as excitation rises — the Stage-A PBS reject port, and + /// the default: on this bench the geometry is settled by construction. + RejectedComplement, + /// Detector brightens with excitation (a transmitted-port tap). + Direct, +} + +impl DetectorGeometry { + pub const VARIANTS: [Self; 2] = [Self::RejectedComplement, Self::Direct]; + + /// Named by what the operator can *observe*, not by optics jargon: the + /// question the setting actually asks is which way the photodiode reading + /// moves when the light reaching the sample gets brighter. + pub fn name(self) -> &'static str { + match self { + Self::RejectedComplement => "REJECT PORT (PD falls as light rises)", + Self::Direct => "DIRECT (PD rises with light)", + } + } + + pub fn from_name(name: &str) -> Option { + Self::VARIANTS.into_iter().find(|kind| kind.name() == name) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TransferFit { + /// DAC code at the excitation minimum. + pub v_null_dac: f64, + /// DAC codes from `v_null` to the excitation maximum (quarter wave). + pub v_pi_dac: f64, + /// Detector volts at the excitation null (`p0`). + pub offset_volts: f64, + /// Signed detector span across one lobe (`p1`); negative on the reject port. + pub span_volts: f64, + pub rms_residual_volts: f64, + /// Residual as a fraction of the detector span — the headline fit quality. + pub quality: f64, + pub geometry: DetectorGeometry, + /// Mean |ascending − descending| at matched codes, as a fraction of the + /// span. `None` when the sweep ran in one direction only. + pub hysteresis: Option, + /// Fraction of one full lobe (`Vπ` codes) the sweep actually covered. + /// Below ~1 the quarter-wave distance is extrapolated, not measured. + pub lobe_coverage: f64, + /// Points discarded as wild before the final fit. A couple is ordinary; a + /// large share means the sweep, not the model, is the problem. + pub rejected_points: usize, + /// Every measured point, rejected ones included, so the plot shows what was + /// actually seen. + pub points: Vec, +} + +impl TransferFit { + pub fn inversion(&self) -> LobeInversion { + LobeInversion { + v_null_dac: self.v_null_dac, + v_pi_dac: self.v_pi_dac, + } + } + + /// Detector extremum at the excitation null. On the reject port this is the + /// detector *maximum* and a **lower bound** on the total-power anchor + /// `I_tot` — not the anchor itself, because the residual transmitted floor + /// is not separable here (knowledge base §4.4). + pub fn detector_volts_at_null(&self) -> f64 { + self.offset_volts + } + + /// Detector extremum at the excitation maximum. + pub fn detector_volts_at_peak(&self) -> f64 { + self.offset_volts + self.span_volts + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum FitError { + /// Fewer points than parameters can be resolved from. + TooFewPoints { count: usize, minimum: usize }, + /// The detector never moved: no lobe to fit (light blocked, no drive + /// reaching the cell, or the sweep span sits in a flat region). + NoModulation, + /// A fitted lobe exists but no `[V_null, V_null+Vπ]` fits inside the + /// commandable range, so no monotonic branch is usable. + NoLobeInRange { v_pi_dac: f64 }, +} + +impl std::fmt::Display for FitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooFewPoints { count, minimum } => { + write!(f, "only {count} sweep points (minimum {minimum})") + } + Self::NoModulation => f.write_str( + "the detector level did not change across the sweep; check the light path, \ + the HV amplifier, and that the photodiode is connected", + ), + Self::NoLobeInRange { v_pi_dac } => write!( + f, + "fitted Vπ = {v_pi_dac:.0} DAC codes leaves no full lobe inside the max limit; \ + raise the max limit or re-check the drive gain" + ), + } + } +} + +impl std::error::Error for FitError {} + +/// Smallest usable sweep: four points per fitted parameter. +pub const MIN_POINTS: usize = 16; +/// A detector span below this is treated as noise rather than a lobe. +const MIN_SPAN_VOLTS: f64 = 0.01; + +/// Least-squares solution for one candidate quarter wave `w`. +struct Harmonic { + /// Mean level `A`, and the quadrature amplitudes of `cos`/`sin(πc/w)`. + mean: f64, + amplitude: f64, + phase: f64, + sse: f64, +} + +/// Fits `P = A + B·cos(πc/w) + C·sin(πc/w)` for a fixed `w`. +/// +/// This is the whole trick that makes the search one-dimensional. Because +/// `sin²(x) = (1 − cos 2x)/2`, the lobe model +/// `p0 + p1·sin²(π(c − v)/(2w))` is *exactly* a constant plus one sinusoid of +/// period `2w` — and a sinusoid of known period is **linear** in its +/// quadrature components. So `V_null` (a phase) and both amplitudes drop out +/// of a 3×3 normal-equation solve, and only `Vπ` is ever searched. No seeding +/// from measured extrema, which is what fails once a sweep spans several +/// lobes and the global extrema sit periods apart. +fn solve_harmonic(points: &[SweepPoint], w: f64) -> Harmonic { + let n = points.len() as f64; + let (mut s_c, mut s_s, mut s_cc, mut s_ss, mut s_cs) = (0.0, 0.0, 0.0, 0.0, 0.0); + let (mut s_y, mut s_yc, mut s_ys) = (0.0, 0.0, 0.0); + for point in points { + let theta = PI * f64::from(point.code) / w; + let (sin, cos) = theta.sin_cos(); + s_c += cos; + s_s += sin; + s_cc += cos * cos; + s_ss += sin * sin; + s_cs += cos * sin; + s_y += point.volts; + s_yc += point.volts * cos; + s_ys += point.volts * sin; + } + // Symmetric 3×3 normal equations for (A, B, C), solved by cofactors. + let m = [[n, s_c, s_s], [s_c, s_cc, s_cs], [s_s, s_cs, s_ss]]; + let rhs = [s_y, s_yc, s_ys]; + let cofactor = [ + m[1][1] * m[2][2] - m[1][2] * m[2][1], + m[1][2] * m[2][0] - m[1][0] * m[2][2], + m[1][0] * m[2][1] - m[1][1] * m[2][0], + ]; + let determinant = m[0][0] * cofactor[0] + m[0][1] * cofactor[1] + m[0][2] * cofactor[2]; + if determinant.abs() < 1e-12 { + return Harmonic { + mean: s_y / n, + amplitude: 0.0, + phase: 0.0, + sse: f64::MAX, + }; + } + let solve = |column: usize| { + let mut augmented = m; + for row in 0..3 { + augmented[row][column] = rhs[row]; + } + (augmented[0][0] * (augmented[1][1] * augmented[2][2] - augmented[1][2] * augmented[2][1]) + - augmented[0][1] + * (augmented[1][0] * augmented[2][2] - augmented[1][2] * augmented[2][0]) + + augmented[0][2] + * (augmented[1][0] * augmented[2][1] - augmented[1][1] * augmented[2][0])) + / determinant + }; + let (a, b, c) = (solve(0), solve(1), solve(2)); + let sse = points + .iter() + .map(|point| { + let theta = PI * f64::from(point.code) / w; + let residual = point.volts - (a + b * theta.cos() + c * theta.sin()); + residual * residual + }) + .sum(); + Harmonic { + mean: a, + amplitude: b.hypot(c), + phase: c.atan2(b), + sse, + } +} + +/// Golden-section minimisation of `f` on `[lo, hi]`, used one axis at a time. +fn golden_min(lo: f64, hi: f64, tolerance: f64, f: impl Fn(f64) -> f64) -> f64 { + const INV_PHI: f64 = 0.618_033_988_749_895; + let (mut lo, mut hi) = (lo, hi); + let mut c = hi - (hi - lo) * INV_PHI; + let mut d = lo + (hi - lo) * INV_PHI; + let (mut fc, mut fd) = (f(c), f(d)); + while (hi - lo) > tolerance { + if fc < fd { + hi = d; + d = c; + fd = fc; + c = hi - (hi - lo) * INV_PHI; + fc = f(c); + } else { + lo = c; + c = d; + fc = fd; + d = lo + (hi - lo) * INV_PHI; + fd = f(d); + } + } + 0.5 * (lo + hi) +} + +/// Mean of the points at each distinct code, smoothed over three neighbours, so +/// the seed extrema are not chosen by a single noisy sample. +fn smoothed_profile(points: &[SweepPoint]) -> Vec<(f64, f64)> { + let mut codes: Vec = points.iter().map(|point| point.code).collect(); + codes.sort_unstable(); + codes.dedup(); + let means: Vec<(f64, f64)> = codes + .iter() + .map(|&code| { + let matching: Vec = points + .iter() + .filter(|point| point.code == code) + .map(|point| point.volts) + .collect(); + ( + f64::from(code), + matching.iter().sum::() / matching.len() as f64, + ) + }) + .collect(); + (0..means.len()) + .map(|index| { + let lo = index.saturating_sub(1); + let hi = (index + 2).min(means.len()); + let window = &means[lo..hi]; + ( + means[index].0, + window.iter().map(|(_, v)| v).sum::() / window.len() as f64, + ) + }) + .collect() +} + +/// Shifts `v` by whole lobe periods to the **lowest** null whose lobe +/// `[v, v + w]` fits inside `0..=max_code`. +/// +/// A sweep across several periods finds several equally valid nulls, so the +/// choice needs a rule the operator can predict rather than a nearest-match. +/// The lowest one drives the Pockels cell at the smallest codes — least +/// voltage across the crystal, most headroom under the max limit. +fn select_lobe(v: f64, w: f64, max_code: f64) -> Option { + // Sub-code precision is meaningless on a 12-bit DAC, so a null fitted a + // hair below 0 (or a peak a hair past the ceiling) is snapped into range + // rather than refused — otherwise a lobe nulling exactly at code 0 fails + // on fit noise alone. + const TOLERANCE: f64 = 1.0; + // The model repeats every `2w` in code, and `v + kw` for odd `k` is the + // same branch mirrored, so stepping by `2w` enumerates every null. + let period = 2.0 * w; + let mut candidate = v - period * ((v / period).floor() + 1.0); + while candidate <= max_code + TOLERANCE { + if candidate >= -TOLERANCE && candidate + w <= max_code + TOLERANCE { + return Some(candidate.clamp(0.0, (max_code - w).max(0.0))); + } + candidate += period; + } + None +} + +/// Mean |ascending − descending| at codes visited in both directions, as a +/// fraction of the detector span. +fn hysteresis_fraction(points: &[SweepPoint], span: f64) -> Option { + let mut differences = Vec::new(); + for up in points + .iter() + .filter(|point| point.direction == Direction::Ascending) + { + if let Some(down) = points + .iter() + .find(|point| point.direction == Direction::Descending && point.code == up.code) + { + differences.push((up.volts - down.volts).abs()); + } + } + if differences.is_empty() || span.abs() < f64::EPSILON { + return None; + } + Some(differences.iter().sum::() / differences.len() as f64 / span.abs()) +} + +/// Scans the quarter wave over every period the sweep could resolve, then +/// refines. Returns the best `(Vπ, harmonic)`. +fn fit_period(points: &[SweepPoint], swept_span: f64) -> Option<(f64, Harmonic)> { + // From four samples per lobe (below that the lobe is aliased) out to a + // lobe twice the swept span (a barely-curved arc). Log-spaced, because a + // fixed step wastes resolution at long periods and misses short ones. + let point_spacing = swept_span / points.len().max(2) as f64; + let w_min = (2.0 * point_spacing).max(1.0); + let w_max = (2.0 * swept_span).max(w_min * 1.5); + const SCAN_STEPS: usize = 600; + let log_step = (w_max / w_min).ln() / SCAN_STEPS as f64; + let mut best: Option<(f64, f64)> = None; // (sse, w) + for step in 0..=SCAN_STEPS { + let w = w_min * (log_step * step as f64).exp(); + let sse = solve_harmonic(points, w).sse; + if best.is_none_or(|(previous, _)| sse < previous) { + best = Some((sse, w)); + } + } + let (_, coarse_w) = best?; + // Refine inside one scan cell, where the SSE is unimodal. + let cell = coarse_w * log_step; + let w = golden_min( + (coarse_w - cell).max(w_min * 0.5), + coarse_w + cell, + 1e-3, + |candidate| solve_harmonic(points, candidate).sse, + ); + let harmonic = solve_harmonic(points, w); + Some((w, harmonic)) +} + +/// Points whose residual against `harmonic` is not wildly out of family. +/// +/// The cut is on the **median** absolute residual, not the mean or the +/// standard deviation: those are themselves dragged out by the very points +/// being looked for. `6 × median` is roughly 4σ for Gaussian noise, so ordinary +/// scatter survives untouched and only genuine strays are dropped. +fn without_outliers(points: &[SweepPoint], w: f64, harmonic: &Harmonic) -> Vec { + let residual = |point: &SweepPoint| { + let theta = PI * f64::from(point.code) / w; + point.volts - (harmonic.mean + harmonic.amplitude * (theta - harmonic.phase).cos()) + }; + let mut magnitudes: Vec = points.iter().map(|point| residual(point).abs()).collect(); + magnitudes.sort_by(f64::total_cmp); + let median = magnitudes[magnitudes.len() / 2]; + if median <= 0.0 { + return points.to_vec(); + } + let limit = 6.0 * median; + points + .iter() + .filter(|point| residual(point).abs() <= limit) + .copied() + .collect() +} + +/// Fits the lobe. `max_code` is the highest commandable DAC code (the drive's +/// max limit), which constrains which branch can be used; `geometry` resolves +/// the null/peak ambiguity the data cannot (see the module docs). +pub fn fit_transfer( + points: &[SweepPoint], + max_code: f64, + geometry: DetectorGeometry, +) -> Result { + if points.len() < MIN_POINTS { + return Err(FitError::TooFewPoints { + count: points.len(), + minimum: MIN_POINTS, + }); + } + let profile = smoothed_profile(points); + let min_volts = profile.iter().map(|(_, v)| *v).fold(f64::MAX, f64::min); + let max_volts = profile.iter().map(|(_, v)| *v).fold(f64::MIN, f64::max); + if max_volts - min_volts < MIN_SPAN_VOLTS { + return Err(FitError::NoModulation); + } + + let swept_lo = profile.first().map(|(code, _)| *code).unwrap_or(0.0); + let swept_hi = profile.last().map(|(code, _)| *code).unwrap_or(max_code); + let swept_span = (swept_hi - swept_lo).max(1.0); + + // A single stray point — one window caught mid-settle, one stream hiccup — + // barely moves the fitted period but inflates the RMS residual several + // fold. Fit once, drop the points the fit says are wild, and fit again on + // what is left, so the reported residual describes the curve rather than + // the worst sample. + let (w, harmonic, rejected_points) = { + let first = fit_period(points, swept_span).ok_or(FitError::NoModulation)?; + let kept = without_outliers(points, first.0, &first.1); + if kept.len() < points.len() && kept.len() >= MIN_POINTS { + match fit_period(&kept, swept_span) { + Some((w, harmonic)) => (w, harmonic, points.len() - kept.len()), + None => (first.0, first.1, 0), + } + } else { + (first.0, first.1, 0) + } + }; + // `A + R·cos(θ − φ)` with `θ = πc/w` is the same curve as + // `p0 + p1·sin²(π(c − v)/(2w))` with `|p1| = 2R`. Which of the two signs + // of `p1` applies — and therefore whether the null sits at the phase or a + // quarter wave past it — is the geometry question the data cannot answer. + let radius = harmonic.amplitude; + let (v, p0, p1) = match geometry { + DetectorGeometry::RejectedComplement => ( + harmonic.phase * w / PI, + harmonic.mean + radius, + -2.0 * radius, + ), + DetectorGeometry::Direct => ( + harmonic.phase * w / PI + w, + harmonic.mean - radius, + 2.0 * radius, + ), + }; + if p1.abs() < MIN_SPAN_VOLTS { + return Err(FitError::NoModulation); + } + + let v_null = select_lobe(v, w, max_code).ok_or(FitError::NoLobeInRange { v_pi_dac: w })?; + + // Over the points the fit actually used: dividing the kept residual by the + // full count would flatter the number. + let rms = (harmonic.sse / (points.len() - rejected_points).max(1) as f64).sqrt(); + Ok(TransferFit { + v_null_dac: v_null, + v_pi_dac: w, + offset_volts: p0, + span_volts: p1, + rms_residual_volts: rms, + quality: rms / p1.abs(), + geometry, + hysteresis: hysteresis_fraction(points, p1), + lobe_coverage: swept_span / w, + rejected_points, + // Every measured point is kept for the plot, rejected ones included: + // seeing the strays next to the fit is how the operator judges it. + points: points.to_vec(), + }) +} + +/// Ascending then descending sweep codes over `0..=max_code`. +pub fn sweep_codes( + max_code: u16, + points_per_pass: usize, + both_directions: bool, +) -> Vec<(u16, Direction)> { + let points_per_pass = points_per_pass.max(2); + let max_code = max_code.min(DAC_FULL_SCALE); + let ascending: Vec = (0..points_per_pass) + .map(|index| { + (f64::from(max_code) * index as f64 / (points_per_pass - 1) as f64).round() as u16 + }) + .collect(); + let mut codes: Vec<(u16, Direction)> = ascending + .iter() + .map(|&code| (code, Direction::Ascending)) + .collect(); + if both_directions { + codes.extend( + ascending + .iter() + .rev() + .map(|&code| (code, Direction::Descending)), + ); + } + codes +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Synthesizes a sweep of a known lobe as seen through a given port. + /// `noise` is a deterministic zig-zag, not an RNG, so failures reproduce. + fn synthetic_sweep( + v_null: f64, + v_pi: f64, + offset: f64, + span: f64, + max_code: u16, + noise: f64, + both_directions: bool, + ) -> Vec { + let lobe = LobeInversion { + v_null_dac: v_null, + v_pi_dac: v_pi, + }; + sweep_codes(max_code, 49, both_directions) + .into_iter() + .enumerate() + .map(|(index, (code, direction))| { + let u = lobe.u_for_dac(f64::from(code)); + let wobble = if index % 2 == 0 { noise } else { -noise }; + SweepPoint { + code, + direction, + volts: offset + span * u + wobble, + peak_to_peak_volts: 0.002, + clipped: false, + } + }) + .collect() + } + + #[test] + fn recovers_a_known_lobe_from_the_reject_port() { + // Reject port: detector is brightest (2.4 V) at the excitation null. + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.004, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement) + .expect("fits the lobe"); + + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null = {}", + fit.v_null_dac + ); + assert!( + (fit.v_pi_dac - 1_600.0).abs() < 10.0, + "Vπ = {}", + fit.v_pi_dac + ); + assert!(fit.span_volts < 0.0, "reject port darkens with excitation"); + assert!((fit.detector_volts_at_null() - 2.4).abs() < 0.02); + assert!(fit.quality < 0.01, "quality = {}", fit.quality); + // Both directions carry the same synthetic curve, so the only + // difference at matched codes is the alternating wobble. + assert!(fit.hysteresis.expect("both directions") < 0.01); + } + + #[test] + fn recovers_the_same_lobe_from_a_direct_detector() { + // Same physical lobe, opposite port: dim at the null, bright at peak. + let points = synthetic_sweep(300.0, 1_600.0, 0.2, 2.2, 4_095, 0.004, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::Direct).expect("fits the lobe"); + + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null = {}", + fit.v_null_dac + ); + assert!((fit.v_pi_dac - 1_600.0).abs() < 10.0); + assert!(fit.span_volts > 0.0, "direct detector brightens"); + } + + #[test] + fn geometry_selects_between_the_two_equivalent_representations() { + // One curve, two readings. Declaring the wrong port must move V_null by + // exactly a quarter wave — the failure this input exists to prevent. + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, false); + let reject = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + let direct = fit_transfer(&points, 4_095.0, DetectorGeometry::Direct).expect("fits"); + + assert!((reject.v_null_dac - 300.0).abs() < 5.0); + assert!( + ((direct.v_null_dac - reject.v_null_dac).abs() - reject.v_pi_dac).abs() < 10.0, + "direct = {}, reject = {}, Vπ = {}", + direct.v_null_dac, + reject.v_null_dac, + reject.v_pi_dac + ); + // Both describe the measured curve equally well; only the physics + // distinguishes them. + assert!((reject.rms_residual_volts - direct.rms_residual_volts).abs() < 1e-6); + } + + #[test] + fn resolves_a_sweep_spanning_several_lobes() { + // A real Vπ near 860 puts ~2.4 lobes inside the DAC range. Seeding the + // period from the global extrema fails here — they can sit whole + // periods apart — which is why the period is scanned, not seeded. + let points = synthetic_sweep(1_630.0, 860.0, 2.4, -2.2, 4_095, 0.003, true); + let fit = fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement) + .expect("fits a multi-lobe sweep"); + assert!((fit.v_pi_dac - 860.0).abs() < 10.0, "Vπ = {}", fit.v_pi_dac); + // Any null is a valid answer as long as it names a real one and the + // lobe it opens fits inside the range. + let offset = (fit.v_null_dac - 1_630.0).rem_euclid(2.0 * 860.0); + assert!( + offset.min(2.0 * 860.0 - offset) < 10.0, + "V_null = {} is not a null of the swept lobe", + fit.v_null_dac + ); + assert!(fit.v_null_dac >= 0.0 && fit.v_null_dac + fit.v_pi_dac <= 4_095.0); + assert!(fit.quality < 0.01, "quality = {}", fit.quality); + assert!(fit.lobe_coverage > 4.0, "coverage = {}", fit.lobe_coverage); + } + + #[test] + fn a_null_at_code_zero_is_not_lost_to_fit_noise() { + // V_null = 0 fits a hair either side of the rail; snapping sub-code + // slack into range is the difference between a usable calibration and + // a refusal. + let points = synthetic_sweep(0.0, 1_200.0, 2.4, -2.2, 4_095, 0.003, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert!(fit.v_null_dac.abs() < 2.0, "V_null = {}", fit.v_null_dac); + } + + #[test] + fn picks_a_lobe_that_fits_inside_the_max_limit() { + // Null at 2600 with Vπ = 1600 would put peak light at 4200, past the + // rail; the previous null one period down (2600 − 3200 < 0) does not + // fit either, so only a lower branch inside the range is acceptable. + let points = synthetic_sweep(1_000.0, 900.0, 2.4, -2.2, 4_095, 0.002, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert!(fit.v_null_dac >= 0.0); + assert!( + fit.v_null_dac + fit.v_pi_dac <= 4_095.0, + "peak light at {} leaves the rail", + fit.v_null_dac + fit.v_pi_dac + ); + } + + #[test] + fn refuses_a_flat_sweep() { + let points: Vec = sweep_codes(4_095, 49, false) + .into_iter() + .map(|(code, direction)| SweepPoint { + code, + direction, + volts: 1.5, + peak_to_peak_volts: 0.001, + clipped: false, + }) + .collect(); + assert_eq!( + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement), + Err(FitError::NoModulation) + ); + } + + #[test] + fn refuses_too_few_points() { + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, false); + assert!(matches!( + fit_transfer(&points[..4], 4_095.0, DetectorGeometry::RejectedComplement), + Err(FitError::TooFewPoints { .. }) + )); + } + + #[test] + fn reports_hysteresis_between_the_two_passes() { + // Descending runs 20 mV below ascending: a real hysteresis signature. + let mut points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.0, true); + for point in &mut points { + if point.direction == Direction::Descending { + point.volts -= 0.02; + } + } + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + let hysteresis = fit.hysteresis.expect("both directions"); + assert!( + (hysteresis - 0.02 / 2.2).abs() < 1e-3, + "hysteresis = {hysteresis}" + ); + } + + #[test] + fn single_direction_sweep_reports_no_hysteresis() { + let points = synthetic_sweep(300.0, 1_600.0, 2.4, -2.2, 4_095, 0.002, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert_eq!(fit.hysteresis, None); + } + + #[test] + fn lobe_coverage_flags_an_extrapolated_quarter_wave() { + // Sweeping only to code 800 with Vπ = 1600 sees half a lobe. + let points = synthetic_sweep(0.0, 1_600.0, 2.4, -2.2, 800, 0.001, false); + let fit = + fit_transfer(&points, 4_095.0, DetectorGeometry::RejectedComplement).expect("fits"); + assert!(fit.lobe_coverage < 0.75, "coverage = {}", fit.lobe_coverage); + } + + #[test] + fn sweep_codes_span_the_range_in_both_directions() { + let codes = sweep_codes(4_000, 5, true); + let ascending: Vec = codes + .iter() + .filter(|(_, direction)| *direction == Direction::Ascending) + .map(|(code, _)| *code) + .collect(); + assert_eq!(ascending, [0, 1_000, 2_000, 3_000, 4_000]); + let descending: Vec = codes + .iter() + .filter(|(_, direction)| *direction == Direction::Descending) + .map(|(code, _)| *code) + .collect(); + assert_eq!(descending, [4_000, 3_000, 2_000, 1_000, 0]); + assert_eq!(sweep_codes(4_000, 5, false).len(), 5); + } +} diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 4c744ce..a77c6f2 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -1,10 +1,11 @@ //! Stage-A laser modulation control. //! //! Drives the laser modulation input (Hermit J23, `DAC1.4`/address 3) through -//! the firmware 0.3.0 `MOD` command. One power slider (DAC code) whose upper -//! bound is a user-set safety cap, a mode select (constant / sine / square) -//! with frequency and a lower threshold for the periodic modes — and every -//! accepted change is transferred to the Teensy immediately, no Apply button. +//! the firmware 0.3.0 `MOD` command. Two orthogonal settings define a drive: +//! the method selects a manually entered or optically calibrated DAC band, +//! while the mode selects the waveform that fills that band. A separate max +//! limit is the hard DAC ceiling for every drive. Every accepted change is +//! transferred to the Teensy immediately, with no Apply button. //! //! **Frame-independent by design.** The host only calls `process_frame()` //! while camera frames flow, so nothing here depends on it: connecting is a @@ -19,7 +20,10 @@ //! (`stage-a-controller` ADR 002): disconnecting does NOT switch the //! modulation off — drag the power slider to 0 to drive 0 V. -use std::collections::BTreeMap; +mod calibration; +mod waveform; + +use std::collections::{BTreeMap, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -28,55 +32,248 @@ use std::time::{Duration, Instant}; use augur_plugin_api::{ export_plugin, EventStoreHandle, ExecutionMode, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, - HostViewRegistry, PathDialogKind, Plugin, PluginFrame, SettingItem, SettingKind, - SettingsSchema, SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, - TableDatasetV1, TableSchema, TableValueType, + HostViewRegistry, PathDialogKind, Plugin, PluginControlContext, PluginControlSnapshot, + PluginFrame, PluginRuntimeRole, PluginServiceOutcome, PluginServiceReply, PluginServiceRequest, + Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, + SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, + TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{Command, MockController, StageAClient, Transport}; +use stage_a_io::{Command, DeviceEvent, MockController, StageAClient, Transport}; +use stage_a_plugin_contract::{ + A1AcquisitionConfigV1, ClientId, ConnectionStateV1, ControllerStateV1, FreshnessV1, LeaseId, + LeaseSnapshotV1, ModulationCommandV1, ModulationRequestV1, ModulationResponseV1, + ModulationStateV1, ModulationTargetV1, OwnerInstanceId, PhotodiodeLevelV1, PhotodiodeSummaryV1, + RequestOutcomeV1, ResponseCommonV1, RunId, SemanticRevision, ServiceErrorCodeV1, + ServiceErrorV1, SynchronizationV1, UnsyncedReasonV1, WaveformV1, CONTRACT_VERSION_V1, + CTX_STAGE_A_MODULATION_STATE_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + PLUGIN_ID_STAGE_A_MODULATION, PLUGIN_ID_STAGE_A_PHOTODIODE, + SERVICE_STAGE_A_MODULATION_CONTROL_V1, +}; const STATUS_DATASET_ID: &str = "stage-a-modulation.status"; const STATUS_VIEW_ID: &str = "stage-a-modulation.status.view"; +const CURVE_DATASET_ID: &str = "stage-a-modulation.transfer-curve"; +const CURVE_VIEW_ID: &str = "stage-a-modulation.transfer-curve.view"; + +/// Codes measured per sweep pass. 49 points over the full range put a sample +/// every ~85 codes, ~19 per lobe at a typical Vπ of 860. +const SWEEP_POINTS_PER_PASS: usize = 49; +/// Samples the detector must have taken *after* a code was commanded before its +/// window counts as settled. At the firmware's 20 kSa/s that is 100 ms — enough +/// for the HV amplifier and the cell to arrive, proven from the sample clock +/// rather than assumed from a timer. +const SETTLE_SAMPLES: u64 = 2_000; +/// Give up on a point if no settled level arrives within this long. A stalled +/// photodiode stream must abort the sweep, not hang it. +const POINT_TIMEOUT: Duration = Duration::from_secs(5); +/// Warn (never block) above this residual, as a fraction of the detector span. +/// A clean bench sits near 1 %; a stray point or two reaches ~10 % while `Vπ` +/// stays good, which is why this warns rather than refuses. +const WARN_QUALITY: f64 = 0.05; +/// Warn above this ascending/descending disagreement, as a fraction of the span. +const WARN_HYSTERESIS: f64 = 0.05; const MAX_DAC_CODE: i64 = 4_095; const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); const DEVICE_LOOP_TICK: Duration = Duration::from_millis(10); +/// After this many serial requests failing in a row the device thread declares +/// the link dead and exits, so the owner can reap it and reconnect. A wedged +/// link that stays "up" otherwise swallows every queued command while the +/// settings UI keeps responding. +const DEVICE_MAX_CONSECUTIVE_ERRORS: u32 = 5; +/// Minimum spacing between automatic reconnect attempts after the device +/// thread died. +const RECONNECT_BACKOFF_MS: u64 = 2_000; +const REQUEST_CACHE_LIMIT: usize = 256; +const MIN_LEASE_TTL_MS: u64 = 250; +const MAX_LEASE_TTL_MS: u64 = 60_000; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Mode { Const, + /// Pure DAC sine (DAC_SINE): the firmware synthesises a sinusoid directly in + /// DAC codes. The optical output is the non-linear `sin²` of this drive. Sine, Square, + /// OPTICAL_LOG_SINE: the DAC is warped so the *optical* output is a + /// log-intensity sine (the clean A1 target). Requires the lobe inversion. + OpticalLogSine, + /// OPTICAL_LINEAR_SINE: the DAC is warped so the optical output is a + /// linear-intensity sine. + OpticalLinearSine, } impl Mode { - const VARIANTS: [Mode; 3] = [Mode::Const, Mode::Sine, Mode::Square]; + const VARIANTS: [Mode; 5] = [ + Mode::Const, + Mode::Sine, + Mode::Square, + Mode::OpticalLogSine, + Mode::OpticalLinearSine, + ]; fn name(self) -> &'static str { match self { Self::Const => "CONST", - Self::Sine => "SINE", + Self::Sine => "DAC_SINE", Self::Square => "SQUARE", + Self::OpticalLogSine => "OPTICAL_LOG_SINE", + Self::OpticalLinearSine => "OPTICAL_LINEAR_SINE", } } fn from_name(name: &str) -> Option { + // Accept the historical "SINE" alias for the pure DAC sine. + if name == "SINE" { + return Some(Self::Sine); + } Self::VARIANTS.into_iter().find(|m| m.name() == name) } fn is_periodic(self) -> bool { !matches!(self, Self::Const) } + + /// Firmware `wave` token. Optical modes upload a warp table and share the + /// `WARP` playback path. + fn wire_wave(self) -> &'static str { + match self { + Self::Const => "CONST", + Self::Sine => "SINE", + Self::Square => "SQUARE", + Self::OpticalLogSine | Self::OpticalLinearSine => "WARP", + } + } + + fn optical_target(self) -> Option { + match self { + Self::OpticalLogSine => Some(waveform::OpticalTarget::LogSine), + Self::OpticalLinearSine => Some(waveform::OpticalTarget::LinearSine), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DriveMethod { + Manual, + Calibrated, +} + +impl DriveMethod { + const VARIANTS: [Self; 2] = [Self::Manual, Self::Calibrated]; + + fn name(self) -> &'static str { + match self { + Self::Manual => "MANUAL", + Self::Calibrated => "CALIBRATED", + } + } + + fn from_name(name: &str) -> Option { + Self::VARIANTS + .into_iter() + .find(|method| method.name() == name) + } } /// State the device thread reports back for the UI (status entries, table). -#[derive(Default)] struct DeviceState { connected: bool, firmware: String, + capabilities: Vec, board_code: Option, board_mod: String, + /// Structured board-echoed modulation (`mod_wave`/`mod_level`/`mod_min`/ + /// `mod_freq_mhz` reply fields). Lets the published snapshot expose the + /// *operator-armed* drive to consumers (A1 derives its fallback + /// modulation period from it) — UI-driven MOD commands never populate the + /// service-path `acknowledged` target. + board_wave: Option, + board_level: Option, + board_min: Option, + board_freq_millihz: Option, last_error: Option, + controller_state: ControllerStateV1, + requested: Option, + acknowledged: Option, + last_response: Option, + last_device_update_unix_ms: u64, +} + +impl Default for DeviceState { + fn default() -> Self { + Self { + connected: false, + firmware: String::new(), + capabilities: Vec::new(), + board_code: None, + board_mod: String::new(), + board_wave: None, + board_level: None, + board_min: None, + board_freq_millihz: None, + last_error: None, + controller_state: ControllerStateV1::Unknown, + requested: None, + acknowledged: None, + last_response: None, + last_device_update_unix_ms: 0, + } + } +} + +impl DeviceState { + /// Board-echo view of the armed drive as a contract target (revision 0), + /// for the published snapshot when no service-path acknowledgement + /// exists. WARP (optical) drives report as `Periodic` — the consumers of + /// this fallback only need the modulation frequency. + fn board_echo_target(&self) -> Option { + let wave = self.board_wave.as_deref()?; + let level = || u16::try_from(self.board_level.unwrap_or(0)).unwrap_or(0); + let min = || u16::try_from(self.board_min.unwrap_or(0)).unwrap_or(0); + let waveform = match wave { + "OFF" => WaveformV1::Off, + "CONST" => WaveformV1::Constant { level_dac: level() }, + "SINE" | "WARP" => WaveformV1::Periodic { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Sine, + min_dac: min(), + max_dac: level(), + frequency_millihz: self.board_freq_millihz.unwrap_or(0), + }, + "SQUARE" => WaveformV1::Periodic { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Square, + min_dac: min(), + max_dac: level(), + frequency_millihz: self.board_freq_millihz.unwrap_or(0), + }, + _ => return None, + }; + Some(ModulationTargetV1 { + revision: SemanticRevision(0), + waveform: Some(waveform), + a1_configuration: None, + acquisition_running: self.controller_state == ControllerStateV1::Running, + board_dac_code: self.board_code.and_then(|code| u16::try_from(code).ok()), + firmware_configuration_revision: None, + }) + } +} + +#[derive(Clone)] +struct OperationMeta { + request_id: stage_a_plugin_contract::RequestId, + run_id: Option, + requested_revision: SemanticRevision, + target: ModulationTargetV1, + owner_instance: OwnerInstanceId, +} + +struct PendingOperation { + commands: Vec, + purpose: &'static str, + meta: Option, } /// Everything shared between the plugin (UI thread) and the device thread. @@ -84,8 +281,10 @@ struct SharedLink { state: Mutex, /// Latest not-yet-sent command; newer settings overwrite older ones so /// slider drags coalesce instead of queueing. - pending: Mutex>, + pending: Mutex>, + priority: Mutex>, stop: AtomicBool, + fail_closed_on_stop: AtomicBool, generation: AtomicU64, } @@ -94,7 +293,9 @@ impl SharedLink { Self { state: Mutex::new(DeviceState::default()), pending: Mutex::new(None), + priority: Mutex::new(None), stop: AtomicBool::new(false), + fail_closed_on_stop: AtomicBool::new(false), generation: AtomicU64::new(1), } } @@ -115,7 +316,7 @@ impl MockService { let link = stage_a_io::MockLink::new(); let stop = Arc::new(AtomicBool::new(false)); let thread_stop = Arc::clone(&stop); - let mut controller = MockController::new(link.device_end()); + let mut controller = MockController::new(link.device_end()).with_waveform_extension(); let join = std::thread::Builder::new() .name("stage-a-modulation-mock".into()) .spawn(move || { @@ -167,10 +368,15 @@ fn run_device(mut client: StageAClient, shared: Arc Ok(fields) => { let mut state = shared.state.lock().expect("device state lock"); state.connected = true; + state.controller_state = ControllerStateV1::SafeIdle; state.firmware = fields .get("firmware") .cloned() .unwrap_or_else(|| "unknown".into()); + state.capabilities = fields + .get("capabilities") + .map(|value| value.split(',').map(str::to_owned).collect()) + .unwrap_or_default(); let has_mod = fields .get("capabilities") .is_some_and(|caps| caps.split(',').any(|c| c == "MOD")); @@ -181,6 +387,7 @@ fn run_device(mut client: StageAClient, shared: Arc Err(err) => { let mut state = shared.state.lock().expect("device state lock"); state.connected = false; + state.controller_state = ControllerStateV1::Faulted; state.last_error = Some(format!("HELLO failed: {err}")); shared.bump(); return; @@ -189,18 +396,49 @@ fn run_device(mut client: StageAClient, shared: Arc shared.bump(); let mut last_status = Instant::now() - STATUS_POLL_INTERVAL; + let mut consecutive_errors = 0u32; while !shared.stop.load(Ordering::Relaxed) { - let pending = shared.pending.lock().expect("pending lock").take(); - if let Some(command) = pending { - let result = client.request(&command); - apply_reply(&shared, "MOD", result); + let priority = shared.priority.lock().expect("priority lock").take(); + let pending = priority.or_else(|| shared.pending.lock().expect("pending lock").take()); + if let Some(operation) = pending { + if execute_operation(&mut client, &shared, operation) { + consecutive_errors = 0; + } else { + consecutive_errors += 1; + } } else if last_status.elapsed() >= STATUS_POLL_INTERVAL { last_status = Instant::now(); let result = client.request(&Command::new("STATUS")); - apply_reply(&shared, "STATUS", result); + if result.is_ok() { + consecutive_errors = 0; + } else { + consecutive_errors += 1; + } + apply_status_reply(&shared, "STATUS", result); } else { + if let Ok(events) = client.poll_events() { + apply_device_events(&shared, events); + } std::thread::sleep(DEVICE_LOOP_TICK); } + if consecutive_errors >= DEVICE_MAX_CONSECUTIVE_ERRORS { + // The link is wedged (unplugged cable, stale fd): declare it dead + // so the owner reaps this thread and reconnects, instead of + // silently swallowing every queued command from here on. + let mut state = shared.state.lock().expect("device state lock"); + state.connected = false; + state.controller_state = ControllerStateV1::Faulted; + state.last_error = Some("serial link failed repeatedly — reconnecting".to_owned()); + drop(state); + shared.bump(); + return; + } + } + + if shared.fail_closed_on_stop.load(Ordering::Relaxed) { + let _ = client.request(&Command::new("STOP").field("reason", "owner_shutdown")); + let result = client.request(&Command::new("MOD").field("wave", "OFF")); + apply_status_reply(&shared, "SAFE_OFF", result); } let mut state = shared.state.lock().expect("device state lock"); @@ -208,7 +446,87 @@ fn run_device(mut client: StageAClient, shared: Arc shared.bump(); } -fn apply_reply( +/// Runs one queued operation; returns whether every command succeeded. +fn execute_operation( + client: &mut StageAClient, + shared: &SharedLink, + operation: PendingOperation, +) -> bool { + let mut merged = BTreeMap::new(); + let mut error = None; + for command in &operation.commands { + match client.request(command) { + Ok(fields) => merged.extend(fields), + Err(err) => { + error = Some(err.to_string()); + break; + } + } + if let Ok(events) = client.poll_events() { + apply_device_events(shared, events); + } + } + + let mut state = shared.state.lock().expect("device state lock"); + let succeeded = error.is_none(); + if let Some(message) = error { + state.last_error = Some(format!("{}: {message}", operation.purpose)); + if let Some(meta) = operation.meta { + state.last_response = Some(ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: meta.request_id, + owner_instance: meta.owner_instance, + run_id: meta.run_id, + requested_revision: Some(meta.requested_revision), + acknowledged_revision: state.acknowledged.as_ref().map(|value| value.revision), + outcome: RequestOutcomeV1::Rejected, + completed_at_unix_ms: Some(now_unix_ms()), + error: Some(ServiceErrorV1 { + code: ServiceErrorCodeV1::DeviceRejected, + message, + retryable: false, + }), + }, + controller_state: state.controller_state, + acknowledged_target: state.acknowledged.clone(), + }); + } + } else { + apply_reply_fields(&mut state, &merged); + state.last_error = None; + if let Some(meta) = operation.meta { + let mut acknowledged = meta.target; + acknowledged.board_dac_code = + state.board_code.and_then(|code| u16::try_from(code).ok()); + acknowledged.firmware_configuration_revision = merged + .get("rev") + .and_then(|value| value.parse::().ok()); + state.acknowledged = Some(acknowledged.clone()); + state.last_response = Some(ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: meta.request_id, + owner_instance: meta.owner_instance, + run_id: meta.run_id, + requested_revision: Some(meta.requested_revision), + acknowledged_revision: Some(meta.requested_revision), + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + controller_state: state.controller_state, + acknowledged_target: Some(acknowledged), + }); + } + } + state.last_device_update_unix_ms = now_unix_ms(); + drop(state); + shared.bump(); + succeeded +} + +fn apply_status_reply( shared: &SharedLink, purpose: &str, result: Result, stage_a_io::ClientError>, @@ -216,32 +534,76 @@ fn apply_reply( let mut state = shared.state.lock().expect("device state lock"); match result { Ok(fields) => { - if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { - state.board_code = Some(code); - } - if let Some(wave) = fields.get("mod_wave") { - let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); - let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); - let freq_mhz = fields - .get("mod_freq_mhz") - .and_then(|v| v.parse::().ok()) - .unwrap_or(0.0); - state.board_mod = if wave == "SINE" || wave == "SQUARE" { - format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) - } else { - format!("{wave} level={level}") - }; - } + apply_reply_fields(&mut state, &fields); if purpose == "MOD" { state.last_error = None; } } Err(err) => state.last_error = Some(format!("{purpose}: {err}")), } + state.last_device_update_unix_ms = now_unix_ms(); drop(state); shared.bump(); } +fn apply_reply_fields(state: &mut DeviceState, fields: &BTreeMap) { + if let Some(code) = fields.get("code").and_then(|v| v.parse::().ok()) { + state.board_code = Some(code); + } + if let Some(controller) = fields.get("state") { + state.controller_state = match controller.as_str() { + "SAFE_IDLE" => ControllerStateV1::SafeIdle, + "CONFIGURED" => ControllerStateV1::Configured, + "RUNNING" => ControllerStateV1::Running, + _ => ControllerStateV1::Unknown, + }; + } + if let Some(wave) = fields.get("mod_wave") { + let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); + let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); + let freq_mhz = fields + .get("mod_freq_mhz") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.0); + state.board_mod = if wave == "SINE" || wave == "SQUARE" { + format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) + } else { + format!("{wave} level={level}") + }; + state.board_wave = Some(wave.clone()); + state.board_level = fields.get("mod_level").and_then(|v| v.parse().ok()); + state.board_min = fields.get("mod_min").and_then(|v| v.parse().ok()); + state.board_freq_millihz = fields.get("mod_freq_mhz").and_then(|v| v.parse().ok()); + } +} + +fn apply_device_events(shared: &SharedLink, events: Vec) { + let fault = events.into_iter().find_map(|event| match event { + DeviceEvent::Async { name, fields } if name == "FAULT" => Some( + fields + .get("code") + .cloned() + .unwrap_or_else(|| "unknown".into()), + ), + _ => None, + }); + if let Some(code) = fault { + let mut state = shared.state.lock().expect("device state lock"); + state.controller_state = ControllerStateV1::Faulted; + state.last_error = Some(format!("controller fault: {code}")); + state.last_device_update_unix_ms = now_unix_ms(); + drop(state); + shared.bump(); + } +} + +fn now_unix_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0) +} + /// One validated protocol step: the exact MOD command plus how long to hold /// it before advancing. #[derive(Debug, Clone, PartialEq)] @@ -342,6 +704,11 @@ fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { } else { let mode = Mode::from_name(&wave) .ok_or_else(|| context("wave must be OFF, CONST, SINE, or SQUARE"))?; + if mode.optical_target().is_some() { + return Err(context( + "optical warp modes are not available in TOML protocol steps; drive them from the modulation UI", + )); + } let level = step .get("level") .and_then(|value| value.as_integer()) @@ -350,7 +717,7 @@ fn parse_protocol(text: &str) -> Result<(Vec, usize), String> { return Err(context("level must be between 0 and 4095")); } let mut command = Command::new("MOD") - .field("wave", mode.name()) + .field("wave", mode.wire_wave()) .field("level", level); let summary; if mode.is_periodic() { @@ -409,7 +776,11 @@ fn run_protocol( progress.step_index = step_index + 1; progress.summary = step.summary.clone(); } - *shared.pending.lock().expect("pending lock") = Some(step.command.clone()); + *shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![step.command.clone()], + purpose: "PROTOCOL", + meta: None, + }); shared.bump(); next_deadline += step.duration; while Instant::now() < next_deadline { @@ -428,8 +799,92 @@ fn run_protocol( shared.bump(); } +/// A transfer-curve sweep in flight. One point at a time: command a settled +/// `CONST` code, wait for a photodiode window that *starts* after the command, +/// record it, move on. +struct CalibrationSweep { + /// Remaining `(code, direction)` steps, and the points collected so far. + steps: Vec<(u16, calibration::Direction)>, + index: usize, + points: Vec, + /// Detector sample index when the current code was commanded. A level only + /// counts once its window begins after this plus [`SETTLE_SAMPLES`], which + /// needs no shared clock and tolerates any tick jitter. + commanded_at_sample: Option, + /// Wall-clock guard for a stream that stops delivering entirely. + point_started: Instant, + /// Drive to restore when the sweep ends, however it ends. + restore: Option, + /// Max limit in force when the sweep started; the fit's branch constraint. + max_code: u16, +} + +/// Forwards momentary button presses across the host's UI-mirror → live-worker +/// settings snapshot. A click arrives as `true` on the clicked instance; the +/// other instance only ever sees the snapshot value from `get_setting`, so the +/// press is transported as a monotonic counter and a counter advance counts as +/// one press edge. The first counter a fresh instance sees is adopted silently +/// so a reloaded worker does not replay old presses (ADR 010). +/// +/// The baseline is tracked separately from the counter: folding the two +/// together makes a fresh worker mistake the operator's *first* real press for +/// its initial sight of the counter and swallow it. +#[derive(Debug, Default, Clone, Copy)] +struct PressLatch { + counter: u64, + seen: Option, +} + +impl PressLatch { + /// Interprets a settings write to this button; returns true on a press edge. + fn accept(&mut self, value: &Value) -> bool { + if value.as_bool() == Some(true) { + self.counter += 1; + self.seen = Some(self.counter); + return true; + } + let Some(incoming) = value.as_u64() else { + return false; + }; + match self.seen { + None => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + false + } + Some(seen) if incoming > seen => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + true + } + Some(_) => false, + } + } + + fn value(&self) -> Value { + json!(self.counter) + } +} + +impl CalibrationSweep { + fn total(&self) -> usize { + self.steps.len() + } + + fn current(&self) -> Option<(u16, calibration::Direction)> { + self.steps.get(self.index).copied() + } +} + pub struct StageAModulationPlugin { enabled: bool, + runtime_role: PluginRuntimeRole, + effects_allowed: bool, + owner_instance: OwnerInstanceId, + lease: Option, + deferred_release_request: Option, + deferred_release_ack_published: bool, + request_cache: VecDeque<(PluginServiceRequest, PluginServiceReply)>, link: Option, shared: Arc, protocol: Option, @@ -439,16 +894,73 @@ pub struct StageAModulationPlugin { max_level: i64, level: i64, min_level: i64, + method: DriveMethod, mode: Mode, frequency_hz: f64, + // -- optical drive inversion (OPTICAL_* modes) -- + /// Requested optical log-modulation depth `a = ln(I_max / I_min)`. + depth_a: f64, + /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. + /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. + operating_point: f64, + /// DAC code at the excitation minimum of one monotonic Pockels lobe. + v_null_dac: i64, + /// DAC-code quarter-wave distance from `v_null` to the excitation maximum. + v_pi_dac: i64, + // -- measured transfer calibration -- + /// Bench detector geometry. Not inferable from a sweep — see + /// [`calibration::DetectorGeometry`]. + detector_geometry: calibration::DetectorGeometry, + /// Sweep in flight, ticked from `process_control`. + sweep: Option, + /// Last completed fit, awaiting review and an explicit apply. + fit: Option, + /// Set once a fit has been applied to `v_null_dac`/`v_pi_dac`; published on + /// the contract so a consumer's sidecar can cite the inversion in use. + calibration_id: Option, + /// Directory for the archived calibration record; empty means "apply the + /// fit but do not archive it". + calibration_dir: String, + /// Operator-visible outcome of the last sweep or apply. + calibration_status: String, + /// Momentary calibration buttons, forwarded mirror → worker (ADR 010). + press_measure: PressLatch, + press_apply: PressLatch, protocol_path: String, last_error: Option, + /// Last automatic reconnect attempt after the device thread died, for the + /// watchdog backoff in `apply_execution_context`. + last_reconnect_ms: u64, + /// Last `protocol_run` value this instance saw. On the UI mirror this is + /// the operator's request (exported through `get_setting`); everywhere it + /// gates actions to value transitions, because the host re-applies the + /// full settings snapshot on every sync. + protocol_requested: bool, +} + +#[derive(Clone)] +struct ControlLease { + lease_id: LeaseId, + holder: ClientId, + run_id: Option, + expires_at_unix_ms: u64, } impl Default for StageAModulationPlugin { fn default() -> Self { Self { enabled: false, + runtime_role: PluginRuntimeRole::UiMirror, + effects_allowed: false, + owner_instance: OwnerInstanceId::new(format!( + "modulation-{}-{}", + std::process::id(), + now_unix_ms() + )), + lease: None, + deferred_release_request: None, + deferred_release_ack_published: false, + request_cache: VecDeque::new(), link: None, shared: Arc::new(SharedLink::new()), protocol: None, @@ -457,10 +969,25 @@ impl Default for StageAModulationPlugin { max_level: MAX_DAC_CODE, level: 0, min_level: 0, + method: DriveMethod::Manual, mode: Mode::Const, frequency_hz: 10.0, + depth_a: 0.5, + operating_point: 0.5, + v_null_dac: 0, + v_pi_dac: 2_048, + detector_geometry: calibration::DetectorGeometry::RejectedComplement, + sweep: None, + fit: None, + calibration_id: None, + calibration_dir: String::new(), + calibration_status: String::new(), + press_measure: PressLatch::default(), + press_apply: PressLatch::default(), protocol_path: String::new(), last_error: None, + last_reconnect_ms: 0, + protocol_requested: false, } } } @@ -470,10 +997,18 @@ impl StageAModulationPlugin { if self.link.is_some() { return; } + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + self.last_error = Some("connection deferred: hardware effects are not allowed".into()); + return; + } self.last_error = None; *self.shared.state.lock().expect("device state lock") = DeviceState::default(); *self.shared.pending.lock().expect("pending lock") = None; + *self.shared.priority.lock().expect("priority lock") = None; self.shared.stop.store(false, Ordering::Relaxed); + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); self.shared.bump(); let shared = Arc::clone(&self.shared); @@ -571,850 +1106,3901 @@ impl StageAModulationPlugin { self.shared.bump(); } + fn lobe_inversion(&self) -> waveform::LobeInversion { + waveform::LobeInversion { + v_null_dac: self.v_null_dac as f64, + v_pi_dac: self.v_pi_dac as f64, + } + } + + /// Resolves the selected method into the DAC band used by every waveform. + /// The third value is the constant-mode operating code. + fn dac_band(&self) -> Result<(i64, i64, i64), String> { + if self.method == DriveMethod::Manual { + let hi = self.level.clamp(0, self.max_level); + return Ok((self.min_level.clamp(0, hi), hi, hi)); + } + + let u_k = self.operating_point; + let a = self.depth_a; + if !u_k.is_finite() || u_k <= 0.0 || u_k > 1.0 { + return Err("operating point must be in (0, 1]".into()); + } + let inversion = self.lobe_inversion(); + if !inversion.v_pi_dac.is_finite() || inversion.v_pi_dac <= 0.0 { + return Err("Vπ must be finite and positive".into()); + } + + // Constant hold at I_k modulates nothing: no ±a/2 headroom applies, so + // the full (0, 1] range of u_k is expressible (I_k = 1 holds exactly at + // V_null + Vπ). Requiring the modulated band here silently froze the + // drive at the last accepted code whenever u_k·e^{a/2} exceeded 1. + if self.mode == Mode::Const { + let hold = inversion.dac_for_u(u_k).round() as i64; + if hold < 0 { + return Err(format!( + "calibrated hold code {hold} is below 0; re-measure V_null/Vπ" + )); + } + if hold > self.max_level { + return Err(format!( + "calibrated hold {hold} exceeds the max limit {}; raise the max limit or lower I_k / Vπ", + self.max_level + )); + } + return Ok((hold, hold, hold)); + } + + if !a.is_finite() || a <= 0.0 { + return Err("optical depth a must be finite and positive".into()); + } + let u_lo = u_k * (-0.5 * a).exp(); + let u_hi = u_k * (0.5 * a).exp(); + if u_hi > 1.0 { + return Err(format!( + "calibrated optical peak u = {u_hi:.3} exceeds the lobe ceiling; lower a or I_k" + )); + } + + let lo = inversion.dac_for_u(u_lo).round() as i64; + let hi = inversion.dac_for_u(u_hi).round() as i64; + let hold = inversion.dac_for_u(u_k).round() as i64; + if lo < 0 { + return Err(format!( + "calibrated lower DAC code {lo} is below 0; re-measure V_null/Vπ" + )); + } + if hi > self.max_level { + return Err(format!( + "calibrated peak {hi} exceeds the max limit {}; raise the max limit or lower a / I_k / Vπ", + self.max_level + )); + } + Ok((lo, hi, hold)) + } + + fn optical_drive(&self, target: waveform::OpticalTarget) -> waveform::OpticalDrive { + let inversion = self.lobe_inversion(); + match self.method { + DriveMethod::Manual => { + let hi = self.level.clamp(0, self.max_level); + let lo = self.min_level.clamp(0, hi); + waveform::OpticalDrive::from_dac_band(target, inversion, lo as f64, hi as f64) + } + DriveMethod::Calibrated => waveform::OpticalDrive { + target, + depth_a: self.depth_a, + operating_point: self.operating_point, + inversion, + }, + } + } + + /// Builds the single MOD command carrying the complete current drive + /// settings (mode, method, band, frequency). Shared by the operator path + /// (`send_modulation`) and the leased `SetOpticalDepth` service command. + fn drive_command(&self) -> Result { + let (lo, hi, hold) = self.dac_band()?; + let freq_mhz = (self.frequency_hz.clamp(0.01, 2_000.0) * 1_000.0).round() as i64; + Ok(match self.mode { + Mode::Const => Command::new("MOD") + .field("wave", "CONST") + .field("level", hold), + Mode::Sine | Mode::Square => Command::new("MOD") + .field("wave", self.mode.wire_wave()) + .field("level", hi) + .field("min", lo) + .field("freq_mhz", freq_mhz), + Mode::OpticalLogSine | Mode::OpticalLinearSine => { + let target = self + .mode + .optical_target() + .expect("optical modes have a target"); + // Validate the drive locally; the firmware rebuilds the same + // table from compact parameters because a full table does not + // fit on one command line. + self.optical_warp_table(target) + .map_err(|error| format!("optical drive: {error}"))?; + let drive = self.optical_drive(target); + Command::new("MOD") + .field("wave", "WARP") + .field("freq_mhz", freq_mhz) + .field("target", optical_target_token(target)) + .field("a_milli", (drive.depth_a * 1_000.0).round() as i64) + .field( + "u_k_milli", + (drive.operating_point * 1_000.0).round() as i64, + ) + .field("v_null", self.v_null_dac) + .field("v_pi", self.v_pi_dac) + } + }) + } + /// Queues one MOD command carrying the complete current drive settings; /// newer changes overwrite queued ones (drag coalescing). + /// + /// Silent while another owner holds the DAC. Besides an automation lease + /// that now includes a calibration sweep: the host re-applies the *whole* + /// settings snapshot on every sync, and most handlers here call this + /// unconditionally, so without the guard every sync would re-arm the + /// operator's drive on top of the code the sweep just commanded — the + /// sweep would measure the armed waveform instead of its own staircase. fn send_modulation(&mut self) { - if self.link.is_none() { + if self.link.is_none() || self.lease.is_some() || self.sweep.is_some() { return; } - let level = self.level.clamp(0, self.max_level); - let mut command = Command::new("MOD") - .field("wave", self.mode.name()) - .field("level", level); - if self.mode.is_periodic() { - let freq_mhz = (self.frequency_hz.clamp(0.01, 2_000.0) * 1_000.0).round() as i64; - command = command - .field("min", self.min_level.clamp(0, level)) - .field("freq_mhz", freq_mhz); - } - *self.shared.pending.lock().expect("pending lock") = Some(command); + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + self.last_error = Some(format!("drive rejected: {error}")); + return; + } + }; + self.last_error = None; + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); } - #[cfg(test)] - fn device_connected(&self) -> bool { - self.shared - .state - .lock() - .map(|state| state.connected) - .unwrap_or(false) + /// Reject a settings update before it can leave the UI showing a drive + /// that was never sent to the board. + fn validate_drive(&mut self) -> Result<(), String> { + match self.drive_command() { + Ok(_) => { + self.last_error = None; + Ok(()) + } + Err(error) => { + self.last_error = Some(format!("drive rejected: {error}")); + Err(error) + } + } } - fn commanded_summary(&self) -> String { - if self.mode.is_periodic() { - format!( - "{} {}..{} @ {:.3} Hz", - self.mode.name(), - self.min_level, - self.level, - self.frequency_hz - ) - } else { - format!("{} level={}", self.mode.name(), self.level) + /// Builds the DAC warp table for the current optical drive settings. The + /// max limit is the hard ceiling; absolute lobe codes cannot be rescaled + /// without distorting the target, so an over-limit drive is refused. + fn optical_warp_table(&self, target: waveform::OpticalTarget) -> Result, String> { + let table = self + .optical_drive(target) + .warp_table() + .map_err(|error| error.to_string())?; + let peak = table.iter().copied().max().unwrap_or(0); + if i64::from(peak) > self.max_level { + return Err(format!( + "optical peak {peak} exceeds the max limit {}; raise the max limit or lower the \ + operating band / a / I_k / Vπ", + self.max_level + )); } + Ok(table) } - fn status_dataset(&self) -> TableDatasetV1 { - let state = self.shared.state.lock().expect("device state lock"); - let connection = if state.connected { - format!("connected ({})", state.firmware) - } else if self.connect_requested { - "connecting…".into() - } else { - "disconnected".into() - }; - let board_code = state - .board_code - .map_or_else(|| "—".into(), |code| code.to_string()); - let error = state - .last_error - .clone() - .or_else(|| self.last_error.clone()) - .unwrap_or_default(); - let board_mod = if state.board_mod.is_empty() { - "—".to_owned() - } else { - state.board_mod.clone() - }; - drop(state); - let text_column = |id: &str, value: String| TableColumnData { - column_id: id.to_owned(), - values: TableColumnValues::String(vec![value]), - }; - TableDatasetV1 { - columns: vec![ - text_column("state", connection), - text_column("commanded", self.commanded_summary()), - text_column("board_mod", board_mod), - text_column("board_code", board_code), - text_column("error", error), - ], - } + // ---- measured transfer calibration ---- + + /// Whether the calibration buttons can be offered, from **mirrored** + /// settings only. + /// + /// `settings_schema()` is rendered by the UI mirror, which never owns the + /// device link, a lease, a sweep, or a fit — those live on the live worker. + /// Gating `enabled` on any of them disables the button permanently. So the + /// affordance uses the one prerequisite the mirror does know (the operator + /// asked to connect) and the authoritative interlocks stay worker-side in + /// [`Self::calibration_blocker`], reported through the status entries the + /// host takes from the worker. + fn calibration_offered(&self) -> bool { + self.connect_requested } - fn status_schema(&self) -> TableSchema { - let column = |id: &str, title: &str| TableColumn { - id: id.to_owned(), - title: title.to_owned(), - value_type: TableValueType::String, - }; - TableSchema { - columns: vec![ - column("state", "State"), - column("commanded", "Commanded drive"), - column("board_mod", "Board modulation"), - column("board_code", "Board DAC code"), - column("error", "Last error"), - ], - ..TableSchema::default() + /// Why a sweep cannot start right now, if it cannot. + fn calibration_blocker(&self) -> Option { + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + return Some("hardware effects are not allowed on this instance".into()); + } + if self.link.is_none() { + return Some("connect the command port first".into()); } + if self.lease.is_some() { + // A1 owns the drive under a lease; two owners stepping the same DAC + // would interleave silently. + return Some("the drive is leased by an automation client".into()); + } + if self.protocol_active() { + return Some("a protocol is running".into()); + } + None } -} -fn open_serial(port_hint: &str) -> Result, String> { - if port_hint == "auto" { - // The dual-serial Teensy enumerates two ports and only the command - // port answers HELLO — probe until one does. - let candidates = serial_ports(); - if candidates.is_empty() { - return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + /// Starts a sweep, remembering the drive to restore afterwards. + fn start_calibration_sweep(&mut self) { + if let Some(blocker) = self.calibration_blocker() { + self.calibration_status = format!("sweep refused: {blocker}"); + return; } - let mut failures = Vec::new(); - for path in &candidates { - match probe_command_port(path) { - // Restore the client's default reply timeout after probing. - Ok(client) => return Ok(client.with_reply_timeout(Duration::from_millis(500))), - Err(err) => failures.push(format!("{path}: {err}")), - } + let max_code = self.max_level.clamp(0, MAX_DAC_CODE) as u16; + if max_code < 2 { + self.calibration_status = "sweep refused: the max limit leaves no range".into(); + return; } - return Err(format!( - "no Teensy command port answered HELLO ({})", - failures.join("; ") - )); + self.sweep = Some(CalibrationSweep { + steps: calibration::sweep_codes(max_code, SWEEP_POINTS_PER_PASS, true), + index: 0, + points: Vec::new(), + commanded_at_sample: None, + point_started: Instant::now(), + // Restoring the drive the operator had armed is part of the + // measurement contract: a sweep must leave the bench as it found it. + restore: self.drive_command().ok(), + max_code, + }); + self.fit = None; + self.calibration_status = "sweep starting…".into(); } - open_path(port_hint) -} -fn open_path(path: &str) -> Result, String> { - let transport = - stage_a_io::SerialTransport::open(path, 115_200, std::time::Duration::from_millis(20)) - .map_err(|err| err.to_string())?; - Ok(StageAClient::new(transport)) -} + /// Ends the sweep and hands the DAC back to the armed drive. + fn finish_calibration_sweep(&mut self, status: String) { + // Clear the sweep first: it is what silences `send_modulation`. + let restore = self.sweep.take().and_then(|sweep| sweep.restore); + self.calibration_status = status; + if self.link.is_some() { + // Prefer the *current* settings — drive changes made during the + // sweep were withheld from the board, and this is where they land. + // The command captured at the start is the fallback for settings + // that no longer form a valid drive. + if self.drive_command().is_ok() { + self.send_modulation(); + } else if let Some(command) = restore { + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); + } + } + self.shared.bump(); + } -/// Opens `path` and sends HELLO with a short timeout: only the Teensy -/// command port replies (the photodiode stream port never answers). -fn probe_command_port(path: &str) -> Result, String> { - let mut client = open_path(path)?.with_reply_timeout(Duration::from_millis(300)); - client - .request(&Command::new("HELLO").field("protocol", 1)) - .map_err(|err| err.to_string())?; - Ok(client) -} + /// Queues one settled `CONST` code, bypassing the drive builder: a sweep + /// deliberately visits codes the armed drive would refuse. + fn command_sweep_code(&mut self, code: u16) { + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![Command::new("MOD") + .field("wave", "CONST") + .field("level", i64::from(code))], + purpose: "MOD", + meta: None, + }); + } -fn serial_ports() -> Vec { - stage_a_io::transport::available_port_names() - .into_iter() - // macOS lists each device twice; use the callout (cu.*) node only. - .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) - .collect() -} + /// One tick of the sweep. `level` is the newest photodiode reading, if any. + fn drive_calibration(&mut self, level: Option) { + if self.sweep.is_none() { + return; + } + if let Some(blocker) = self.calibration_blocker() { + self.finish_calibration_sweep(format!("sweep aborted: {blocker}")); + return; + } + let Some(level) = level else { + if self + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started.elapsed() > POINT_TIMEOUT) + { + self.finish_calibration_sweep( + "sweep aborted: no photodiode level (connect the photodiode plugin)".into(), + ); + } + return; + }; -/// The exact variant list the settings schema shows for the port enum — the -/// host exchanges enum settings as indices into this list. Real ports carry -/// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; -/// only the leading path is the value. -fn port_variants() -> Vec { - let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; - for (name, label) in stage_a_io::transport::available_ports_with_labels() { - if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { - continue; + let Some((code, direction)) = self.sweep.as_ref().and_then(CalibrationSweep::current) + else { + self.complete_calibration_sweep(); + return; + }; + + // Command the point once, then wait for a window that began after it. + let commanded_at = match self.sweep.as_ref().expect("sweep").commanded_at_sample { + Some(sample) => sample, + None => { + self.command_sweep_code(code); + let sweep = self.sweep.as_mut().expect("sweep"); + sweep.commanded_at_sample = Some(level.end_sample_index); + sweep.point_started = Instant::now(); + self.calibration_status = format!( + "sweeping {}/{}…", + self.sweep.as_ref().expect("sweep").index + 1, + self.sweep.as_ref().expect("sweep").total() + ); + return; + } + }; + + let window_start = level.end_sample_index.saturating_sub(level.sample_count); + if window_start < commanded_at + SETTLE_SAMPLES { + if self.sweep.as_ref().expect("sweep").point_started.elapsed() > POINT_TIMEOUT { + self.finish_calibration_sweep( + "sweep aborted: the photodiode stream stalled".into(), + ); + } + return; } - variants.push(match label { - Some(label) => format!("{name} ({label})"), - None => name, + + let sweep = self.sweep.as_mut().expect("sweep"); + sweep.points.push(calibration::SweepPoint { + code, + direction, + volts: level.mean_volts, + peak_to_peak_volts: level.peak_to_peak_volts, + clipped: level.clipped, }); + sweep.index += 1; + sweep.commanded_at_sample = None; + if sweep.index >= sweep.steps.len() { + self.complete_calibration_sweep(); + } } - variants -} -/// The path part of a port variant; the parenthesised USB label is display-only. -fn variant_path(variant: &str) -> &str { - variant.split_whitespace().next().unwrap_or(variant) -} + /// Fits the collected points and leaves the result awaiting an explicit + /// apply — a bad fit silently retargeting the drive is the dangerous case. + fn complete_calibration_sweep(&mut self) { + let Some(sweep) = self.sweep.as_ref() else { + return; + }; + let points = sweep.points.clone(); + let max_code = f64::from(sweep.max_code); + match calibration::fit_transfer(&points, max_code, self.detector_geometry) { + Ok(fit) => { + let status = format!( + "V_null {:.0} Vπ {:.0} span {:.3} V residual {:.1}%{}{} ({:.1} lobes)", + fit.v_null_dac, + fit.v_pi_dac, + fit.span_volts.abs(), + fit.quality * 100.0, + fit.hysteresis + .map(|value| format!(" hysteresis {:.1}%", value * 100.0)) + .unwrap_or_default(), + if fit.rejected_points > 0 { + format!(" {} dropped", fit.rejected_points) + } else { + String::new() + }, + fit.lobe_coverage, + ); + self.fit = Some(fit); + self.finish_calibration_sweep(status); + } + Err(error) => { + self.fit = None; + self.finish_calibration_sweep(format!("fit failed: {error}")); + } + } + } -/// Host enum widgets send the selected index; string names are also accepted -/// (tests, saved configs). -fn enum_choice(value: &Value, variants: &[String]) -> Result { - if let Some(index) = value.as_u64() { - return variants - .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) - .cloned() - .ok_or_else(|| format!("enum index {index} out of range")); + /// Things worth the operator's attention before trusting a fit. Compare + /// them against the transfer-curve plot. + /// + /// Deliberately warnings and not blocks. The only condition that makes a + /// fit meaningless — no full lobe inside the commandable range — is already + /// refused by [`calibration::fit_transfer`] itself, so there is no second + /// fit to reject here. Everything below is a judgement the operator makes + /// against the plot: a single stray sample can push the residual past any + /// threshold while `Vπ` stays accurate to a few codes, so blocking on it + /// would withhold a good calibration for a bad reason. + fn fit_warnings(&self) -> Vec { + let Some(fit) = self.fit.as_ref() else { + return Vec::new(); + }; + let mut warnings = Vec::new(); + if fit.quality > WARN_QUALITY { + warnings.push(format!( + "residual is {:.1}% of the detector span — check the fit against the points \ + in the transfer-curve plot before trusting Vπ", + fit.quality * 100.0 + )); + } + if fit.rejected_points > 0 { + warnings.push(format!( + "{} of {} points were wild and left out of the fit", + fit.rejected_points, + fit.points.len() + )); + } + if let Some(hysteresis) = fit.hysteresis.filter(|value| *value > WARN_HYSTERESIS) { + warnings.push(format!( + "up and down passes differ by {:.1}% of the span — the cell is drifting or \ + the settle time is too short", + hysteresis * 100.0 + )); + } + let clipped = fit.points.iter().filter(|point| point.clipped).count(); + if clipped > 0 { + warnings.push(format!( + "{clipped} points clipped the ADC; the extremum they sit on is not where the \ + fit thinks it is — add attenuation and re-measure" + )); + } + warnings } - value - .as_str() - .map(str::to_owned) - .ok_or_else(|| "expected an enum index or name".to_owned()) -} -impl Plugin for StageAModulationPlugin { - fn name(&self) -> &'static str { - "Stage-A Modulation" + /// Applies the reviewed fit to `V_null`/`Vπ` and archives the record. + fn apply_calibration_fit(&mut self) { + let Some(fit) = self.fit.clone() else { + self.calibration_status = "nothing to apply: measure a transfer curve first".into(); + return; + }; + let previous = (self.v_null_dac, self.v_pi_dac); + self.v_null_dac = fit.v_null_dac.round().clamp(0.0, MAX_DAC_CODE as f64) as i64; + self.v_pi_dac = fit.v_pi_dac.round().clamp(1.0, MAX_DAC_CODE as f64) as i64; + // The applied lobe must still produce a legal drive; a calibration that + // cannot be armed is not an improvement. + if let Err(error) = self.validate_drive() { + self.v_null_dac = previous.0; + self.v_pi_dac = previous.1; + self.calibration_status = format!("not applied: {error}"); + return; + } + let calibration_id = format!("pockels-{}", timestamp_slug()); + let archived = match self.archive_calibration(&calibration_id, &fit) { + Ok(Some(path)) => format!(", archived to {path}"), + Ok(None) => ", not archived (no calibration folder set)".into(), + Err(error) => format!(", archive failed: {error}"), + }; + self.calibration_id = Some(calibration_id); + self.calibration_status = format!( + "applied V_null {} / Vπ {}{archived}", + self.v_null_dac, self.v_pi_dac + ); + self.send_modulation(); + self.shared.bump(); } - fn description(&self) -> &'static str { - "Laser modulation control on the Teensy command port: capped power slider, constant/sine/square with frequency, applied immediately; shows the DAC code the board reports." + /// Writes the calibration record. A calibration is named and never + /// silently overwritten (knowledge base §4.6). + fn archive_calibration( + &self, + calibration_id: &str, + fit: &calibration::TransferFit, + ) -> Result, String> { + if self.calibration_dir.trim().is_empty() { + return Ok(None); + } + let directory = std::path::Path::new(self.calibration_dir.trim()); + std::fs::create_dir_all(directory) + .map_err(|error| format!("creating {}: {error}", directory.display()))?; + let path = directory.join(format!("{calibration_id}.json")); + let record = json!({ + "calibration_id": calibration_id, + "port": self.port_hint, + "max_level": self.max_level, + "detector_geometry": fit.geometry.name(), + "v_null_dac": fit.v_null_dac, + "v_pi_dac": fit.v_pi_dac, + "detector_volts_at_null": fit.detector_volts_at_null(), + "detector_volts_at_peak": fit.detector_volts_at_peak(), + "span_volts": fit.span_volts, + "rms_residual_volts": fit.rms_residual_volts, + "quality": fit.quality, + "hysteresis": fit.hysteresis, + "lobe_coverage": fit.lobe_coverage, + "rejected_points": fit.rejected_points, + "anchor_note": "detector_volts_at_null is a lower bound on the total-power \ + anchor I_tot, not the anchor: on the reject port the residual \ + transmitted floor is not separable from it", + "points": fit + .points + .iter() + .map(|point| json!({ + "code": point.code, + "direction": point.direction.label(), + "volts": point.volts, + "peak_to_peak_volts": point.peak_to_peak_volts, + "clipped": point.clipped, + })) + .collect::>(), + }); + let encoded = serde_json::to_vec_pretty(&record) + .map_err(|error| format!("encoding the calibration record: {error}"))?; + std::fs::write(&path, encoded) + .map_err(|error| format!("writing {}: {error}", path.display()))?; + Ok(Some(path.display().to_string())) } - fn enabled(&self) -> bool { - self.enabled + fn lease_snapshot(&self) -> Option { + self.lease.as_ref().map(|lease| LeaseSnapshotV1 { + lease_id: lease.lease_id.clone(), + holder: lease.holder.clone(), + expires_at_unix_ms: lease.expires_at_unix_ms, + run_id: lease.run_id.clone(), + }) } - fn set_enabled(&mut self, enabled: bool) { - self.enabled = enabled; - if !enabled { - self.connect_requested = false; - self.disconnect(); + fn require_lease(&self, request: &ModulationRequestV1) -> Result<(), ServiceErrorV1> { + let lease = self.lease.as_ref().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::LeaseRequired, + "the modulation owner requires an active automation lease", + false, + ) + })?; + if now_unix_ms() > lease.expires_at_unix_ms { + return Err(service_error( + ServiceErrorCodeV1::LeaseExpired, + "the modulation automation lease expired", + false, + )); } + if request.lease_id.as_ref() != Some(&lease.lease_id) || request.requester != lease.holder { + return Err(service_error( + ServiceErrorCodeV1::LeaseMismatch, + "request lease/holder does not match the active lease", + false, + )); + } + if request.run_id != lease.run_id { + return Err(service_error( + ServiceErrorCodeV1::LeaseMismatch, + "request run does not match the leased run", + false, + )); + } + Ok(()) } - fn reset(&mut self) {} - - fn process_frame( - &mut self, - _frame: &PluginFrame<'_>, - _output: &mut HostOutput<'_>, - context: &mut HostContext<'_>, - _event_store: &EventStoreHandle<'_>, - ) { - // Control is settings-driven and works without camera frames. The - // only frame-pass policy: replaying a recording must never keep a - // hardware connection alive. - if context.execution().mode == ExecutionMode::Replay && self.link.is_some() { - self.connect_requested = false; - self.disconnect(); - self.last_error = Some("disconnected: replay mode".into()); + fn requested_revision( + &self, + request: &ModulationRequestV1, + ) -> Result { + let revision = request.requested_revision.ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "state-changing modulation commands require requested_revision", + false, + ) + })?; + let current = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.requested.as_ref().map(|target| target.revision)); + if current.is_some_and(|current| revision <= current) { + return Err(service_error( + ServiceErrorCodeV1::StaleRequest, + "requested_revision must be newer than the current requested state", + false, + )); } + Ok(revision) } - fn settings_schema(&self) -> SettingsSchema { - let port_variants = port_variants(); - let port_default = port_variants - .iter() - .position(|p| variant_path(p) == self.port_hint) - .unwrap_or(0); - let mode_variants: Vec = - Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); - let mode_default = Mode::VARIANTS - .iter() - .position(|m| *m == self.mode) - .unwrap_or(0); - SettingsSchema { - sections: vec![ - SettingsSection { - label: "Laser modulation".into(), - description: Some( - "Tick Connect, then every change is sent to the Teensy immediately — no \ - camera required. The output never exceeds the power slider, the slider \ - never exceeds the max limit. The firmware holds the output when \ - disconnected; drag the slider to 0 to drive 0 V." - .into(), - ), - default_open: true, - items: vec![ - SettingItem { - key: "port".into(), - label: "Port".into(), - tooltip: Some( - "auto (recommended) probes the attached usbmodem ports and picks \ - the one that answers HELLO — the Teensy command port; \ - mock = in-process simulated controller" - .into(), - ), - kind: SettingKind::Enum { - variants: port_variants, - default: port_default, - }, - }, - SettingItem { - key: "connect".into(), - label: "Connect".into(), - tooltip: Some( - "Opens/closes the command port. Connecting never changes the \ - output; disconnecting leaves it held (set-and-hold firmware)." - .into(), - ), - kind: SettingKind::Bool { - default: self.connect_requested, - }, - }, - SettingItem { - key: "level".into(), - label: "Power (DAC code)".into(), - tooltip: Some( - "Output level in DAC codes; peak value for sine/square. \ - Capped by the max limit below. 0 = output off." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.level, - suffix: None, - }, - }, - SettingItem { - key: "max_level".into(), - label: "Max limit (DAC code)".into(), - tooltip: Some( - "Safety cap: the slider cannot go above this. Set it to the \ - highest code the connected device tolerates at J23." - .into(), - ), - kind: SettingKind::I64Drag { - min: 0, - max: MAX_DAC_CODE, - default: self.max_level, - }, - }, - SettingItem { - key: "mode".into(), - label: "Mode".into(), - tooltip: Some("CONST holds the level; SINE/SQUARE modulate".into()), - kind: SettingKind::Enum { - variants: mode_variants, - default: mode_default, - }, - }, - SettingItem { - key: "frequency_hz".into(), - label: "Frequency".into(), - tooltip: Some("Sine/square frequency, 0.01–2000 Hz".into()), - kind: SettingKind::F64Drag { - min: 0.01, - max: 2_000.0, - speed: 1.0, - default: self.frequency_hz, - }, - }, - SettingItem { - key: "min_level".into(), - label: "Min threshold (DAC code)".into(), - tooltip: Some( - "Lower bound for sine/square: the waveform swings between this \ - and the power slider. Ignored in CONST mode." - .into(), - ), - kind: SettingKind::I64Slider { - min: 0, - max: self.max_level, - default: self.min_level, - suffix: None, - }, - }, - ], - }, - SettingsSection { - label: "Protocol".into(), - description: Some( - "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ - [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ - min, frequency_hz. Steps run on an absolute schedule; the last \ - step holds after completion (set-and-hold). Stopping never \ - switches the output off by itself." - .into(), - ), - default_open: false, - items: vec![ - SettingItem { - key: "protocol_path".into(), - label: "Protocol file".into(), - tooltip: Some("TOML protocol file (validated on start).".into()), - kind: SettingKind::Path { - dialog: PathDialogKind::OpenFile, - default: self.protocol_path.clone(), - }, - }, - SettingItem { - key: "protocol_run".into(), - label: "Run protocol".into(), - tooltip: Some( - "Start/stop the loaded protocol. Requires an open connection; \ - manual drive controls stay live and override the current step \ - until the next one begins." - .into(), - ), - kind: SettingKind::Bool { - default: self.protocol_active(), - }, - }, - ], - }, - ], + fn base_target(&self, revision: SemanticRevision) -> ModulationTargetV1 { + let state = self.shared.state.lock().expect("device state lock"); + let mut target = state + .requested + .clone() + .or_else(|| state.acknowledged.clone()) + .unwrap_or(ModulationTargetV1 { + revision, + waveform: None, + a1_configuration: None, + acquisition_running: false, + board_dac_code: None, + firmware_configuration_revision: None, + }); + target.revision = revision; + target.board_dac_code = None; + target.firmware_configuration_revision = None; + target + } + + fn queue_service_operation( + &mut self, + request: &ModulationRequestV1, + target: ModulationTargetV1, + commands: Vec, + purpose: &'static str, + priority: bool, + ) -> Result { + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the Teensy command port is not connected", + true, + )); + } + let revision = target.revision; + let meta = OperationMeta { + request_id: request.request_id, + run_id: request.run_id.clone(), + requested_revision: revision, + target: target.clone(), + owner_instance: self.owner_instance.clone(), + }; + { + let mut state = self.shared.state.lock().expect("device state lock"); + state.requested = Some(target); + } + let operation = PendingOperation { + commands, + purpose, + meta: Some(meta), + }; + if priority { + *self.shared.pending.lock().expect("pending lock") = None; + *self.shared.priority.lock().expect("priority lock") = Some(operation); + } else { + *self.shared.pending.lock().expect("pending lock") = Some(operation); } + self.shared.bump(); + Ok(ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: request.request_id, + owner_instance: self.owner_instance.clone(), + run_id: request.run_id.clone(), + requested_revision: Some(revision), + acknowledged_revision: self + .shared + .state + .lock() + .ok() + .and_then(|state| state.acknowledged.as_ref().map(|value| value.revision)), + outcome: RequestOutcomeV1::InProgress, + completed_at_unix_ms: None, + error: None, + }, + controller_state: self + .shared + .state + .lock() + .map(|state| state.controller_state) + .unwrap_or(ControllerStateV1::Unknown), + acknowledged_target: None, + }) } - fn get_setting(&self, key: &str) -> Option { - match key { - // Enum settings are exchanged as indices into the schema's - // variant list (see the host settings UI). - "port" => { - let index = port_variants() - .iter() - .position(|p| variant_path(p) == self.port_hint) - .unwrap_or(0); - Some(json!(index)) - } - "connect" => Some(json!(self.connect_requested)), - "level" => Some(json!(self.level)), - "max_level" => Some(json!(self.max_level)), - "mode" => { - let index = Mode::VARIANTS - .iter() - .position(|m| *m == self.mode) - .unwrap_or(0); - Some(json!(index)) - } - "frequency_hz" => Some(json!(self.frequency_hz)), - "min_level" => Some(json!(self.min_level)), - "protocol_path" => Some(json!(self.protocol_path)), - "protocol_run" => Some(json!(self.protocol_active())), - _ => None, - } - } - - fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { - match key { - "port" => { - self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); - Ok(()) - } - "connect" => { - let requested = value.as_bool().ok_or("connect must be a boolean")?; - self.connect_requested = requested; - if requested { - self.connect(); - } else { - self.disconnect(); - } - Ok(()) - } - "level" => { - self.level = value - .as_i64() - .ok_or("level must be an integer")? - .clamp(0, self.max_level); - if self.min_level > self.level { - self.min_level = self.level; + fn handle_modulation_command( + &mut self, + request: &ModulationRequestV1, + ) -> Result { + match &request.command { + ModulationCommandV1::Connect => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "connection cannot be changed while leased", + false, + )); } - self.send_modulation(); - Ok(()) + self.connect_requested = true; + self.connect(); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "max_level" => { - self.max_level = value - .as_i64() - .ok_or("max_level must be an integer")? - .clamp(0, MAX_DAC_CODE); - // Lowering the cap below the current level lowers the output. - if self.level > self.max_level { - self.level = self.max_level; - self.send_modulation(); + ModulationCommandV1::Disconnect { safe_off, reason } => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "use ReleaseLease while the owner is leased", + false, + )); } - if self.min_level > self.max_level { - self.min_level = self.max_level; + if *safe_off && self.link.is_some() { + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); } - Ok(()) + self.connect_requested = false; + self.disconnect(); + self.last_error = Some(format!("disconnected by service: {reason}")); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "mode" => { - let mode_names: Vec = - Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); - let name = enum_choice(&value, &mode_names)?; - self.mode = Mode::from_name(&name) - .ok_or_else(|| format!("unknown mode: {name} (CONST/SINE/SQUARE)"))?; - self.send_modulation(); - Ok(()) + ModulationCommandV1::AcquireLease { ttl_ms } => { + let lease_id = request.lease_id.clone().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "AcquireLease requires lease_id", + false, + ) + })?; + if let Some(active) = &self.lease { + if active.lease_id != lease_id || active.holder != request.requester { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "the modulation owner is already leased", + true, + )); + } + } + self.protocol = None; + self.lease = Some(ControlLease { + lease_id, + holder: request.requester.clone(), + run_id: request.run_id.clone(), + expires_at_unix_ms: lease_deadline(*ttl_ms), + }); + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "frequency_hz" => { - let hz = value.as_f64().ok_or("frequency_hz must be a number")?; - self.frequency_hz = hz.clamp(0.01, 2_000.0); - if self.mode.is_periodic() { - self.send_modulation(); + ModulationCommandV1::RenewLease { ttl_ms } => { + self.require_lease(request)?; + if let Some(lease) = &mut self.lease { + lease.expires_at_unix_ms = lease_deadline(*ttl_ms); } - Ok(()) + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "min_level" => { - self.min_level = value - .as_i64() - .ok_or("min_level must be an integer")? - .clamp(0, self.level); - if self.mode.is_periodic() { - self.send_modulation(); + ModulationCommandV1::ReleaseLease { safe_off, reason } => { + self.require_lease(request)?; + if *safe_off { + let revision = request.requested_revision.unwrap_or_else(|| { + let current = self + .shared + .state + .lock() + .ok() + .and_then(|state| { + state.requested.as_ref().map(|value| value.revision.0) + }) + .unwrap_or(0); + SemanticRevision(current.saturating_add(1)) + }); + let mut target = self.base_target(revision); + target.waveform = Some(WaveformV1::Off); + target.acquisition_running = false; + let response = self.queue_service_operation( + request, + target, + vec![ + Command::new("STOP").field("reason", reason.replace(' ', "_")), + Command::new("MOD").field("wave", "OFF"), + ], + "SAFE_OFF", + true, + )?; + self.deferred_release_request = Some(request.request_id); + self.deferred_release_ack_published = false; + return Ok(response); } - Ok(()) + self.lease = None; + self.deferred_release_request = None; + self.shared + .fail_closed_on_stop + .store(false, Ordering::Relaxed); + self.immediate_response(request, RequestOutcomeV1::Applied, None) } - "protocol_path" => { - self.protocol_path = value - .as_str() - .ok_or("protocol_path must be a string")? - .to_owned(); - Ok(()) + ModulationCommandV1::SafeOff { reason } => { + let revision = request.requested_revision.unwrap_or_else(|| { + let current = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.requested.as_ref().map(|value| value.revision.0)) + .unwrap_or(0); + SemanticRevision(current.saturating_add(1)) + }); + let mut target = self.base_target(revision); + target.waveform = Some(WaveformV1::Off); + target.acquisition_running = false; + self.protocol = None; + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); + self.queue_service_operation( + request, + target, + vec![ + Command::new("STOP").field("reason", reason.replace(' ', "_")), + Command::new("MOD").field("wave", "OFF"), + ], + "SAFE_OFF", + true, + ) } - "protocol_run" => { - let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; - // Failures surface through status entries (like `connect`). - if requested { - match self.start_protocol() { - Ok(()) => self.last_error = None, - Err(err) => self.last_error = Some(err), - } - } else { - self.stop_protocol(); + ModulationCommandV1::SetWaveform { waveform } => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.waveform = Some(waveform.clone()); + self.queue_service_operation( + request, + target, + vec![waveform_command(waveform)], + "SET_WAVEFORM", + false, + ) + } + ModulationCommandV1::SetOpticalDepth { depth_a_milli } => { + self.require_lease(request)?; + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the modulation owner is not connected to the device", + false, + )); + } + let depth_a = f64::from(*depth_a_milli) / 1_000.0; + if !(0.01..=6.0).contains(&depth_a) { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + format!("optical depth a={depth_a:.3} outside the supported 0.01..=6.0"), + false, + )); } + // Only the calibrated drive expresses an optical depth; the + // manual DAC band and the constant hold do not. + if self.method == DriveMethod::Manual || self.mode == Mode::Const { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "arm a calibrated periodic/optical drive in the modulation plugin \ + before sweeping the optical depth", + false, + )); + } + let previous = self.depth_a; + self.depth_a = depth_a; + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + self.depth_a = previous; + return Err(service_error( + ServiceErrorCodeV1::DeviceRejected, + format!("optical depth a={depth_a:.3} rejected: {error}"), + false, + )); + } + }; + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); self.shared.bump(); - Ok(()) + self.immediate_response(request, RequestOutcomeV1::Applied, None) + } + ModulationCommandV1::PrepareA1 { configuration } => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.a1_configuration = Some(configuration.clone()); + target.acquisition_running = false; + self.queue_service_operation( + request, + target, + vec![ + Command::new("STOP").field("reason", "prepare_a1"), + a1_config_command(configuration), + ], + "PREPARE_A1", + false, + ) + } + ModulationCommandV1::StartAcquisition => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.acquisition_running = true; + self.queue_service_operation( + request, + target, + vec![Command::new("START")], + "START", + false, + ) + } + ModulationCommandV1::StopAcquisition { reason } => { + self.require_lease(request)?; + let revision = self.requested_revision(request)?; + let mut target = self.base_target(revision); + target.acquisition_running = false; + self.queue_service_operation( + request, + target, + vec![Command::new("STOP").field("reason", reason.replace(' ', "_"))], + "STOP", + false, + ) } - _ => Err(format!("unknown setting: {key}")), } } - fn status_entries(&self) -> Vec { - let mut entries = Vec::new(); + fn immediate_response( + &mut self, + request: &ModulationRequestV1, + outcome: RequestOutcomeV1, + error: Option, + ) -> Result { let state = self.shared.state.lock().expect("device state lock"); - entries.push(StatusEntry::Text(if state.connected { - format!("Modulation: connected ({})", state.firmware) + let response = ModulationResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: request.request_id, + owner_instance: self.owner_instance.clone(), + run_id: request.run_id.clone(), + requested_revision: request.requested_revision, + acknowledged_revision: state.acknowledged.as_ref().map(|value| value.revision), + outcome, + completed_at_unix_ms: Some(now_unix_ms()), + error, + }, + controller_state: state.controller_state, + acknowledged_target: state.acknowledged.clone(), + }; + drop(state); + self.shared + .state + .lock() + .expect("device state lock") + .last_response = Some(response.clone()); + self.shared.bump(); + Ok(response) + } + + fn control_state(&self) -> ModulationStateV1 { + let state = self.shared.state.lock().expect("device state lock"); + let connection = if state.connected { + ConnectionStateV1::Connected { + port_label: self.port_hint.clone(), + firmware_version: Some(state.firmware.clone()), + } + } else if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { + ConnectionStateV1::Faulted { message: error } } else if self.connect_requested { - "Modulation: connecting…".into() + ConnectionStateV1::Connecting } else { - "Modulation: disconnected".into() - })); - if let Some(code) = state.board_code { - entries.push(StatusEntry::Text(format!( - "Board: code={code} ({})", - state.board_mod - ))); - } - if let Some(run) = &self.protocol { - if let Ok(progress) = run.progress.lock() { - entries.push(StatusEntry::Text(if progress.finished { - if progress.stopped { - "Protocol: stopped (last step holds)".into() - } else { - "Protocol: finished (last step holds)".into() - } - } else { - format!( - "Protocol: loop {}/{} step {}/{} — {}", - progress.loop_index, - progress.loops, - progress.step_index, - progress.total_steps, - progress.summary - ) - })); + ConnectionStateV1::Disconnected + }; + let synchronization = match ( + self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + state.requested.as_ref(), + state.acknowledged.as_ref(), + ) { + (Some(run_id), Some(requested), Some(acknowledged)) + if requested.revision == acknowledged.revision => + { + SynchronizationV1::Synced { + run_id, + acknowledged_revision: acknowledged.revision, + stream_epoch: None, + } } + (None, _, _) => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::NoLease, + detail: None, + }, + _ => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::RequestedRevisionNotAcknowledged, + detail: None, + }, + }; + ModulationStateV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: self.owner_instance.clone(), + service_revision: self.shared.generation.load(Ordering::Relaxed), + connection, + capabilities: state.capabilities.clone(), + lease: self.lease_snapshot(), + controller_state: state.controller_state, + active_run_id: self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + requested: state.requested.clone(), + // Service-path acknowledgements win; otherwise expose the + // board-echoed operator-armed drive (revision 0) so consumers + // like A1 can read the modulation frequency without a lease ever + // having existed. + acknowledged: state + .acknowledged + .clone() + .or_else(|| state.board_echo_target()), + synchronization, + last_response: state.last_response.clone(), + freshness: FreshnessV1 { + observed_at_unix_ms: if state.last_device_update_unix_ms == 0 { + now_unix_ms() + } else { + state.last_device_update_unix_ms + }, + valid_for_ms: 1_500, + }, + calibration_id: self.calibration_id.clone(), } - if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { - entries.push(StatusEntry::Text(format!("Error: {error}"))); - } - entries } - fn host_views(&self) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![HostDatasetDescriptor { - id: STATUS_DATASET_ID.into(), - title: "Laser modulation".into(), - kind: HostDatasetKind::TableV1(self.status_schema()), - empty_message: "Modulation control idle.".into(), - display: None, - relations: Vec::new(), - }], - views: vec![HostViewDescriptor { - id: STATUS_VIEW_ID.into(), - title: "Laser modulation".into(), - dataset_id: STATUS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }], - actions: Vec::new(), + fn expire_lease_if_needed(&mut self) { + let expired = self + .lease + .as_ref() + .is_some_and(|lease| now_unix_ms() > lease.expires_at_unix_ms); + if !expired { + return; } + self.protocol = None; + self.shared + .fail_closed_on_stop + .store(true, Ordering::Relaxed); + *self.shared.pending.lock().expect("pending lock") = None; + *self.shared.priority.lock().expect("priority lock") = Some(PendingOperation { + commands: vec![ + Command::new("STOP").field("reason", "lease_expired"), + Command::new("MOD").field("wave", "OFF"), + ], + purpose: "LEASE_EXPIRED_SAFE_OFF", + meta: None, + }); + self.lease = None; + self.last_error = Some("automation lease expired; queued STOP + output off".into()); + self.shared.bump(); } - fn host_view_dataset(&self, dataset_id: &str) -> Option> { - match dataset_id { - STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), - _ => None, + fn advance_deferred_release(&mut self) { + let Some(request_id) = self.deferred_release_request else { + return; + }; + let terminal_applied = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.last_response.clone()) + .is_some_and(|response| { + response.common.request_id == request_id + && response.common.outcome == RequestOutcomeV1::Applied + }); + if !terminal_applied { + return; + } + if self.deferred_release_ack_published { + self.lease = None; + self.deferred_release_request = None; + self.deferred_release_ack_published = false; + self.shared + .fail_closed_on_stop + .store(false, Ordering::Relaxed); + self.shared.bump(); + } else { + // Preserve the lease for one complete snapshot publication so + // the orchestrator can consume the terminal ACK before the owner + // advertises the release. + self.deferred_release_ack_published = true; } } - fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { - match dataset_id { - STATUS_DATASET_ID => self.shared.generation.load(Ordering::Relaxed).max(1), - _ => 0, + fn apply_execution_context(&mut self, execution: &augur_plugin_api::ExecutionContext) { + let allowed = self.runtime_role == PluginRuntimeRole::LiveWorker + && execution.hardware_effects_allowed(); + self.effects_allowed = allowed; + if !allowed { + if self.link.is_some() { + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + } + self.lease = None; + self.deferred_release_request = None; + self.deferred_release_ack_published = false; + return; + } + self.expire_lease_if_needed(); + self.advance_deferred_release(); + // Reap a dead device thread (failed HELLO, wedged serial): a finished + // thread leaves `link` occupied, which both swallows every queued + // command (the settings UI keeps responding while the board holds the + // old waveform) and blocks the auto-reconnect below. + if self + .link + .as_ref() + .and_then(|link| link.join.as_ref()) + .is_some_and(JoinHandle::is_finished) + { + self.link = None; + } + if self.connect_requested && self.link.is_none() { + let now_ms = now_unix_ms(); + if now_ms.saturating_sub(self.last_reconnect_ms) >= RECONNECT_BACKOFF_MS { + self.last_reconnect_ms = now_ms; + self.connect(); + } } } -} -impl Drop for StageAModulationPlugin { - fn drop(&mut self) { - self.disconnect(); + #[cfg(test)] + fn device_connected(&self) -> bool { + self.shared + .state + .lock() + .map(|state| state.connected) + .unwrap_or(false) } -} -export_plugin!(StageAModulationPlugin); + fn commanded_summary(&self) -> String { + match self.dac_band() { + Ok((lo, hi, hold)) if self.mode == Mode::Const => format!( + "{} {} hold={} (band {}..{})", + self.method.name(), + self.mode.name(), + hold, + lo, + hi + ), + Ok((lo, hi, _)) => format!( + "{} {} {}..{} @ {:.3} Hz", + self.method.name(), + self.mode.name(), + lo, + hi, + self.frequency_hz + ), + Err(error) => format!( + "{} {} invalid: {error}", + self.method.name(), + self.mode.name() + ), + } + } -#[cfg(test)] -mod tests { - use super::*; + /// The transfer curve the operator reasons about. Before any sweep it shows + /// the lobe the *configured* `V_null`/`Vπ` claim, on a normalised axis, so + /// the two numbers are legible with no hardware attached; after a fit it + /// shows what was actually measured, in detector volts. + fn curve_dataset(&self) -> Series1dV1 { + let max_code = self.max_level.clamp(1, MAX_DAC_CODE) as f64; + let sample_curve = |scale: f64, offset: f64, inversion: waveform::LobeInversion| { + (0..=256) + .map(|step| { + let code = max_code * f64::from(step) / 256.0; + Series1dPoint { + x: code, + y: offset + scale * inversion.u_for_dac(code), + } + }) + .collect::>() + }; + // Two-point verticals mark the lobe endpoints on whatever y-range the + // rest of the plot spans. + let marker = |name: &str, code: f64, lo: f64, hi: f64| Series1dLine { + name: name.to_owned(), + points: vec![ + Series1dPoint { x: code, y: lo }, + Series1dPoint { x: code, y: hi }, + ], + }; - fn wait_until bool>( - plugin: &StageAModulationPlugin, - timeout: Duration, - done: F, - ) { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if done(plugin) { - return; - } - std::thread::sleep(Duration::from_millis(2)); + let Some(fit) = self.fit.as_ref() else { + let inversion = self.lobe_inversion(); + let mut lines = vec![Series1dLine { + name: "configured lobe".into(), + points: sample_curve(1.0, 0.0, inversion), + }]; + lines.push(marker("V_null", inversion.v_null_dac, 0.0, 1.0)); + lines.push(marker( + "V_null + Vπ", + inversion.v_null_dac + inversion.v_pi_dac, + 0.0, + 1.0, + )); + return Series1dV1 { + x_label: "DAC code".into(), + y_label: "normalised transmission u (not yet measured)".into(), + lines, + }; + }; + + let point_line = |direction: calibration::Direction| Series1dLine { + name: format!("measured {}", direction.label()), + points: fit + .points + .iter() + .filter(|point| point.direction == direction) + .map(|point| Series1dPoint { + x: f64::from(point.code), + y: point.volts, + }) + .collect(), + }; + let (lo, hi) = fit + .points + .iter() + .fold((f64::MAX, f64::MIN), |(lo, hi), point| { + (lo.min(point.volts), hi.max(point.volts)) + }); + let mut lines = vec![ + point_line(calibration::Direction::Ascending), + point_line(calibration::Direction::Descending), + Series1dLine { + name: "fit".into(), + points: sample_curve(fit.span_volts, fit.offset_volts, fit.inversion()), + }, + ]; + // The configured lobe on the fit's own scale: after applying they + // coincide, and any divergence is the un-applied difference. + let configured = self.lobe_inversion(); + if configured != fit.inversion() { + lines.push(Series1dLine { + name: "configured lobe".into(), + points: sample_curve(fit.span_volts, fit.offset_volts, configured), + }); } - panic!("condition not reached within {timeout:?}"); + lines.push(marker("V_null", fit.v_null_dac, lo, hi)); + lines.push(marker("V_null + Vπ", fit.v_null_dac + fit.v_pi_dac, lo, hi)); + Series1dV1 { + x_label: "DAC code".into(), + y_label: "photodiode [V]".into(), + lines, + } + } + + fn status_dataset(&self) -> TableDatasetV1 { + let state = self.shared.state.lock().expect("device state lock"); + let connection = if state.connected { + format!("connected ({})", state.firmware) + } else if self.connect_requested { + "connecting…".into() + } else { + "disconnected".into() + }; + let board_code = state + .board_code + .map_or_else(|| "—".into(), |code| code.to_string()); + let error = state + .last_error + .clone() + .or_else(|| self.last_error.clone()) + .unwrap_or_default(); + let board_mod = if state.board_mod.is_empty() { + "—".to_owned() + } else { + state.board_mod.clone() + }; + drop(state); + let text_column = |id: &str, value: String| TableColumnData { + column_id: id.to_owned(), + values: TableColumnValues::String(vec![value]), + }; + TableDatasetV1 { + columns: vec![ + text_column("state", connection), + text_column("commanded", self.commanded_summary()), + text_column("board_mod", board_mod), + text_column("board_code", board_code), + text_column("error", error), + ], + } + } + + fn status_schema(&self) -> TableSchema { + let column = |id: &str, title: &str| TableColumn { + id: id.to_owned(), + title: title.to_owned(), + value_type: TableValueType::String, + }; + TableSchema { + columns: vec![ + column("state", "State"), + column("commanded", "Commanded drive"), + column("board_mod", "Board modulation"), + column("board_code", "Board DAC code"), + column("error", "Last error"), + ], + ..TableSchema::default() + } + } +} + +/// `YYYYmmdd-HHMMSS` in UTC, from the wall clock alone (no chrono dependency). +fn timestamp_slug() -> String { + let seconds = now_unix_ms() / 1_000; + let (days, time) = (seconds / 86_400, seconds % 86_400); + // Civil-from-days, Howard Hinnant's algorithm, shifted to a 0000-03-01 era. + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let day_of_era = z.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_prime = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_prime + 2) / 5 + 1; + let month = if month_prime < 10 { + month_prime + 3 + } else { + month_prime - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= 2); + format!( + "{year:04}{month:02}{day:02}-{:02}{:02}{:02}", + time / 3_600, + (time % 3_600) / 60, + time % 60 + ) +} + +fn open_serial(port_hint: &str) -> Result, String> { + if port_hint == "auto" { + // The dual-serial Teensy enumerates two ports and only the command + // port answers HELLO — probe until one does. + let candidates = serial_ports(); + if candidates.is_empty() { + return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + } + let mut failures = Vec::new(); + for path in &candidates { + match probe_command_port(path) { + // Restore the client's default reply timeout after probing. + Ok(client) => return Ok(client.with_reply_timeout(Duration::from_millis(500))), + Err(err) => failures.push(format!("{path}: {err}")), + } + } + return Err(format!( + "no Teensy command port answered HELLO ({})", + failures.join("; ") + )); + } + open_path(port_hint) +} + +fn service_error( + code: ServiceErrorCodeV1, + message: impl Into, + retryable: bool, +) -> ServiceErrorV1 { + ServiceErrorV1 { + code, + message: message.into(), + retryable, + } +} + +fn lease_deadline(ttl_ms: u64) -> u64 { + now_unix_ms().saturating_add(ttl_ms.clamp(MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS)) +} + +/// Wire token for the optical target on the `MOD wave=WARP` command. +fn optical_target_token(target: waveform::OpticalTarget) -> &'static str { + match target { + waveform::OpticalTarget::LogSine => "LOG_SINE", + waveform::OpticalTarget::LinearSine => "LINEAR_SINE", + } +} + +fn waveform_command(waveform: &WaveformV1) -> Command { + match waveform { + WaveformV1::Off => Command::new("MOD").field("wave", "OFF"), + WaveformV1::Constant { level_dac } => Command::new("MOD") + .field("wave", "CONST") + .field("level", *level_dac), + WaveformV1::Periodic { + waveform, + min_dac, + max_dac, + frequency_millihz, + } => Command::new("MOD") + .field( + "wave", + match waveform { + stage_a_plugin_contract::PeriodicWaveformV1::Sine => "SINE", + stage_a_plugin_contract::PeriodicWaveformV1::Square => "SQUARE", + }, + ) + .field("level", *max_dac) + .field("min", *min_dac) + .field("freq_mhz", *frequency_millihz), + } +} + +fn a1_config_command(configuration: &A1AcquisitionConfigV1) -> Command { + Command::new("CONFIG") + .field("mode", "A1") + .field( + "wave", + match configuration.waveform { + stage_a_plugin_contract::PeriodicWaveformV1::Sine => "SINE", + stage_a_plugin_contract::PeriodicWaveformV1::Square => "SQUARE", + }, + ) + .field("freq_mhz", configuration.frequency_millihz) + .field("center_dac", configuration.center_dac) + .field("amplitude_dac", configuration.amplitude_dac) + .field("rate_hz", configuration.sample_rate_hz) + .field("block_samples", configuration.block_samples) + .field("raw", u8::from(configuration.emit_raw_samples)) + .field("summary", u8::from(configuration.emit_summary)) +} + +fn accepted_service_reply( + request: &PluginServiceRequest, + response: &ModulationResponseV1, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).unwrap_or(Value::Null), + }, + } +} + +fn rejected_service_reply( + request: &PluginServiceRequest, + code: &str, + message: impl Into, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Rejected { + code: code.into(), + message: message.into(), + }, + } +} + +fn open_path(path: &str) -> Result, String> { + let transport = + stage_a_io::SerialTransport::open(path, 115_200, std::time::Duration::from_millis(20)) + .map_err(|err| err.to_string())?; + Ok(StageAClient::new(transport)) +} + +/// Opens `path` and sends HELLO with a short timeout: only the Teensy +/// command port replies (the photodiode stream port never answers). +fn probe_command_port(path: &str) -> Result, String> { + let mut client = open_path(path)?.with_reply_timeout(Duration::from_millis(300)); + client + .request(&Command::new("HELLO").field("protocol", 1)) + .map_err(|err| err.to_string())?; + Ok(client) +} + +fn serial_ports() -> Vec { + stage_a_io::transport::available_port_names() + .into_iter() + // macOS lists each device twice; use the callout (cu.*) node only. + .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) + .collect() +} + +/// The exact variant list the settings schema shows for the port enum — the +/// host exchanges enum settings as indices into this list. Real ports carry +/// their USB label (e.g. "(Teensyduino Dual Serial)") for recognisability; +/// only the leading path is the value. +fn port_variants() -> Vec { + let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; + for (name, label) in stage_a_io::transport::available_ports_with_labels() { + if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { + continue; + } + variants.push(match label { + Some(label) => format!("{name} ({label})"), + None => name, + }); + } + variants +} + +/// The path part of a port variant; the parenthesised USB label is display-only. +fn variant_path(variant: &str) -> &str { + variant.split_whitespace().next().unwrap_or(variant) +} + +/// Host enum widgets send the selected index; string names are also accepted +/// (tests, saved configs). +fn enum_choice(value: &Value, variants: &[String]) -> Result { + if let Some(index) = value.as_u64() { + return variants + .get(usize::try_from(index).map_err(|_| "index out of range".to_owned())?) + .cloned() + .ok_or_else(|| format!("enum index {index} out of range")); + } + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| "expected an enum index or name".to_owned()) +} + +impl Plugin for StageAModulationPlugin { + fn name(&self) -> &'static str { + "Stage-A Modulation" + } + + fn description(&self) -> &'static str { + "Laser modulation control on the Teensy command port: capped power slider, constant/sine/square with frequency, applied immediately; shows the DAC code the board reports." + } + + fn enabled(&self) -> bool { + self.enabled + } + + fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.connect_requested = false; + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + self.lease = None; + self.deferred_release_request = None; + } + } + + fn set_runtime_role(&mut self, role: PluginRuntimeRole) { + self.runtime_role = role; + if role != PluginRuntimeRole::LiveWorker { + self.effects_allowed = false; + if self.link.is_some() { + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + } + self.lease = None; + self.deferred_release_request = None; + self.deferred_release_ack_published = false; + } + } + + fn reset(&mut self) {} + + fn process_frame( + &mut self, + _frame: &PluginFrame<'_>, + _output: &mut HostOutput<'_>, + context: &mut HostContext<'_>, + _event_store: &EventStoreHandle<'_>, + ) { + // Control is settings-driven and works without camera frames. The + // only frame-pass policy: replaying a recording must never keep a + // hardware connection alive. + if context.execution().mode == ExecutionMode::Replay && self.link.is_some() { + self.connect_requested = false; + self.shared + .fail_closed_on_stop + .store(self.lease.is_some(), Ordering::Relaxed); + self.disconnect(); + self.lease = None; + self.last_error = Some("disconnected: replay mode".into()); + } + } + + fn process_control(&mut self, context: &mut PluginControlContext<'_>) { + let execution = context.execution(); + self.apply_execution_context(&execution); + // The photodiode owner broadcasts its summary to every plugin's inbox, + // so a calibration sweep reads the light with no lease and no request. + let level = context + .inbox() + .snapshots + .iter() + .find(|snapshot| { + snapshot.plugin_id == PLUGIN_ID_STAGE_A_PHOTODIODE + && snapshot.topic == CTX_STAGE_A_PHOTODIODE_SUMMARY_V1 + }) + .and_then(|snapshot| { + serde_json::from_value::(snapshot.payload.clone()).ok() + }) + .and_then(|summary| summary.stream.level); + self.drive_calibration(level); + } + + fn handle_service_request( + &mut self, + request: &PluginServiceRequest, + execution: &augur_plugin_api::ExecutionContext, + ) -> PluginServiceReply { + if let Some(index) = self.request_cache.iter().position(|(previous, _)| { + previous.source_plugin_id == request.source_plugin_id + && previous.request_id == request.request_id + }) { + let (previous, cached_reply) = self.request_cache[index].clone(); + if previous != *request { + return rejected_service_reply( + request, + "request_id_conflict", + "request ID was reused for different modulation payload", + ); + } + let cached_in_progress = match &cached_reply.outcome { + PluginServiceOutcome::Accepted { payload } => serde_json::from_value::< + ModulationResponseV1, + >(payload.clone()) + .is_ok_and(|response| response.common.outcome == RequestOutcomeV1::InProgress), + PluginServiceOutcome::Rejected { .. } => false, + }; + if cached_in_progress { + let terminal = self + .shared + .state + .lock() + .ok() + .and_then(|state| state.last_response.clone()) + .filter(|response| { + response.common.request_id.0 == request.request_id + && response.common.outcome != RequestOutcomeV1::InProgress + }); + if let Some(terminal) = terminal { + let upgraded = accepted_service_reply(request, &terminal); + self.request_cache[index].1 = upgraded.clone(); + return upgraded; + } + } + return cached_reply; + } + + let reply = if request.target_plugin_id != PLUGIN_ID_STAGE_A_MODULATION { + rejected_service_reply(request, "wrong_target", "wrong modulation owner target") + } else if request.service != SERVICE_STAGE_A_MODULATION_CONTROL_V1 { + rejected_service_reply( + request, + "unsupported_service", + format!("unsupported modulation service '{}'", request.service), + ) + } else if self.runtime_role != PluginRuntimeRole::LiveWorker + || !execution.hardware_effects_allowed() + { + rejected_service_reply( + request, + "effects_not_allowed", + "modulation effects are allowed only on the active live worker", + ) + } else { + self.effects_allowed = true; + match serde_json::from_value::(request.payload.clone()) { + Err(err) => rejected_service_reply( + request, + "invalid_payload", + format!("invalid modulation request: {err}"), + ), + Ok(payload) + if payload.contract_version != CONTRACT_VERSION_V1 + || payload.request_id.0 != request.request_id + || payload.requester.as_str() != request.source_plugin_id + || payload + .target_owner_instance + .as_ref() + .is_some_and(|owner| owner != &self.owner_instance) => + { + rejected_service_reply( + request, + "identity_mismatch", + "contract version, request, requester, or owner instance mismatch", + ) + } + Ok(payload) + if payload.issued_at_unix_ms != 0 + && (now_unix_ms().saturating_sub(payload.issued_at_unix_ms) > 120_000 + || payload.issued_at_unix_ms.saturating_sub(now_unix_ms()) + > 30_000) => + { + rejected_service_reply(request, "stale_request", "request timestamp is stale") + } + Ok(payload) => match self.handle_modulation_command(&payload) { + Ok(response) => accepted_service_reply(request, &response), + Err(error) => rejected_service_reply( + request, + &format!("{:?}", error.code).to_ascii_lowercase(), + error.message, + ), + }, + } + }; + self.request_cache + .push_back((request.clone(), reply.clone())); + while self.request_cache.len() > REQUEST_CACHE_LIMIT { + self.request_cache.pop_front(); + } + reply + } + + fn control_snapshots(&self) -> Vec { + vec![PluginControlSnapshot { + plugin_id: PLUGIN_ID_STAGE_A_MODULATION.into(), + topic: CTX_STAGE_A_MODULATION_STATE_V1.into(), + revision: self.shared.generation.load(Ordering::Relaxed).max(1), + payload: serde_json::to_value(self.control_state()).unwrap_or(Value::Null), + }] + } + + fn settings_schema(&self) -> SettingsSchema { + let port_variants = port_variants(); + let port_default = port_variants + .iter() + .position(|p| variant_path(p) == self.port_hint) + .unwrap_or(0); + let method_variants: Vec = DriveMethod::VARIANTS + .iter() + .map(|method| method.name().to_owned()) + .collect(); + let method_default = DriveMethod::VARIANTS + .iter() + .position(|method| *method == self.method) + .unwrap_or(0); + let mode_variants: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let mode_default = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + let mut modulation_items = vec![ + SettingItem { + key: "port".into(), + label: "Port".into(), + tooltip: Some( + "auto (recommended) probes the attached usbmodem ports and picks \ + the one that answers HELLO — the Teensy command port; \ + mock = in-process simulated controller" + .into(), + ), + kind: SettingKind::Enum { + variants: port_variants, + default: port_default, + }, + }, + SettingItem { + key: "connect".into(), + label: "Connect".into(), + tooltip: Some( + "Opens/closes the command port. Connecting never changes the \ + output; disconnecting leaves it held (set-and-hold firmware)." + .into(), + ), + kind: SettingKind::Bool { + default: self.connect_requested, + }, + }, + SettingItem { + key: "max_level".into(), + label: "Max limit (DAC code)".into(), + tooltip: Some( + "Hard ceiling for every drive. No manual or calibrated waveform may \ + produce a DAC code above this value at J23." + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.max_level, + }, + }, + SettingItem { + key: "method".into(), + label: "Drive method".into(), + tooltip: Some( + "MANUAL defines the DAC band with Power and Min threshold. CALIBRATED \ + derives it from V_null, Vπ, I_k, and optical depth a." + .into(), + ), + kind: SettingKind::Enum { + variants: method_variants, + default: method_default, + }, + }, + SettingItem { + key: "mode".into(), + label: "Mode".into(), + tooltip: Some( + "Selects the waveform that fills the method-defined operating band. \ + All five modes are available with both drive methods." + .into(), + ), + kind: SettingKind::Enum { + variants: mode_variants, + default: mode_default, + }, + }, + SettingItem { + key: "frequency_hz".into(), + label: "Frequency".into(), + tooltip: Some("Periodic-waveform frequency, 0.01–2000 Hz".into()), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.frequency_hz, + }, + }, + ]; + match self.method { + DriveMethod::Manual => { + modulation_items.push(SettingItem { + key: "level".into(), + label: "Power (DAC code)".into(), + tooltip: Some( + "Manual peak/operating DAC code. CONST holds this value; periodic \ + modes use it as the upper end of the manual band." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.level, + suffix: None, + }, + }); + modulation_items.push(SettingItem { + key: "min_level".into(), + label: "Min threshold (DAC code)".into(), + tooltip: Some( + "Lower end of the manual DAC band. Ignored by CONST, which holds Power." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: self.max_level, + default: self.min_level, + suffix: None, + }, + }); + } + DriveMethod::Calibrated => { + modulation_items.push(SettingItem { + key: "v_null_dac".into(), + label: "V_null (DAC code at min light)".into(), + tooltip: Some( + "DAC code where excitation light bottoms out (sin² = 0) on one \ + monotonic Pockels lobe. Measure it; do not trust nominal Vπ." + .into(), + ), + kind: SettingKind::I64Drag { + min: 0, + max: MAX_DAC_CODE, + default: self.v_null_dac, + }, + }); + modulation_items.push(SettingItem { + key: "v_pi_dac".into(), + label: "Vπ (DAC codes, null → max light)".into(), + tooltip: Some( + "DAC-code quarter-wave distance from V_null to the excitation \ + maximum. V_null + Vπ must stay within 0..4095." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: MAX_DAC_CODE, + default: self.v_pi_dac, + }, + }); + modulation_items.push(SettingItem { + key: "operating_point".into(), + label: "Operating point I_k (0..1)".into(), + tooltip: Some( + "Calibrated operating illumination as normalised lobe intensity u_k. \ + CONST holds its DAC code; the calibrated band is derived around it." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 1.0, + speed: 0.01, + default: self.operating_point, + }, + }); + modulation_items.push(SettingItem { + key: "depth_a".into(), + label: "Optical depth a".into(), + tooltip: Some( + "Calibrated log-intensity span a = ln(I_max/I_min). Together with I_k \ + it defines the operating band used by every mode." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 6.0, + speed: 0.01, + default: self.depth_a, + }, + }); + } + } + let geometry_variants: Vec = calibration::DetectorGeometry::VARIANTS + .iter() + .map(|geometry| geometry.name().to_owned()) + .collect(); + let geometry_default = calibration::DetectorGeometry::VARIANTS + .iter() + .position(|geometry| *geometry == self.detector_geometry) + .unwrap_or(0); + SettingsSchema { + sections: vec![ + SettingsSection { + label: "Laser modulation".into(), + description: Some( + "Tick Connect, then every change is sent to the Teensy immediately — no \ + camera required. Method selects the operating band; Mode selects its \ + waveform. Max limit is the hard ceiling. The firmware holds the output \ + when disconnected." + .into(), + ), + default_open: true, + items: modulation_items, + }, + SettingsSection { + label: "Calibration".into(), + description: Some( + "Measures the Pockels/PBS transfer curve: steps settled CONST DAC codes \ + across the range while reading the photodiode, then fits V_null and Vπ. \ + Needs the photodiode plugin connected. The sweep restores your armed \ + drive when it finishes, and the fit is never applied without your \ + confirmation. Watch the transfer-curve view." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "detector_geometry".into(), + label: "Detector port".into(), + tooltip: Some( + "Which way the photodiode moves when the light reaching the \ + sample gets brighter. Stage-A's photodiode sits on the PBS \ + reject port and reads the leftover light, I_pd = I_tot − I_exc, \ + so it goes DOWN as the sample gets brighter — that is REJECT \ + PORT, the default. Pick DIRECT only for a detector that watches \ + the sample beam itself. The sweep cannot work this out: a bright \ + and a dark extremum fit the measured curve equally well, and \ + only the optics say which one is zero light on the sample. \ + Choosing wrong puts V_null a quarter wave off." + .into(), + ), + kind: SettingKind::Enum { + variants: geometry_variants, + default: geometry_default, + }, + }, + SettingItem { + key: "calibrate".into(), + label: "Measure transfer curve".into(), + tooltip: Some( + "Sweeps the full range up and back down (~20 s), then fits the \ + lobe. Press again to abort; the armed drive is restored either \ + way. Progress and the result appear in the status lines below." + .into(), + ), + kind: SettingKind::Button { + enabled: self.calibration_offered(), + }, + }, + SettingItem { + key: "calibrate_apply".into(), + label: "Apply to V_null / Vπ".into(), + tooltip: Some( + "Writes the fitted lobe into the calibrated drive settings. \ + Refused, with the reason in the status lines, until a sweep has \ + produced a fit that is good enough to trust: residual within \ + 2 % of the detector span, at least three quarters of a lobe \ + covered, and no clipped point." + .into(), + ), + kind: SettingKind::Button { + enabled: self.calibration_offered(), + }, + }, + SettingItem { + key: "calibration_dir".into(), + label: "Calibration folder (optional)".into(), + tooltip: Some( + "Where the applied calibration record is archived, with its \ + points and fit. Leave empty to apply without archiving." + .into(), + ), + kind: SettingKind::Path { + dialog: PathDialogKind::Directory, + default: self.calibration_dir.clone(), + }, + }, + ], + }, + SettingsSection { + label: "Protocol".into(), + description: Some( + "Timed sequence of MOD steps from a TOML file: `loops = N` plus \ + [[steps]] with duration_s, wave (OFF/CONST/SINE/SQUARE), level, \ + min, frequency_hz. Steps run on an absolute schedule; the last \ + step holds after completion (set-and-hold). Stopping never \ + switches the output off by itself." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "protocol_path".into(), + label: "Protocol file".into(), + tooltip: Some("TOML protocol file (validated on start).".into()), + kind: SettingKind::Path { + dialog: PathDialogKind::OpenFile, + default: self.protocol_path.clone(), + }, + }, + SettingItem { + key: "protocol_run".into(), + label: "Run protocol".into(), + tooltip: Some( + "Start/stop the loaded protocol. Requires an open connection; \ + manual drive controls stay live and override the current step \ + until the next one begins." + .into(), + ), + kind: SettingKind::Bool { + default: if self.runtime_role == PluginRuntimeRole::LiveWorker { + self.protocol_active() + } else { + self.protocol_requested + }, + }, + }, + ], + }, + ], + } + } + + fn get_setting(&self, key: &str) -> Option { + match key { + // Enum settings are exchanged as indices into the schema's + // variant list (see the host settings UI). + "port" => { + let index = port_variants() + .iter() + .position(|p| variant_path(p) == self.port_hint) + .unwrap_or(0); + Some(json!(index)) + } + "connect" => Some(json!(self.connect_requested)), + "level" => Some(json!(self.level)), + "max_level" => Some(json!(self.max_level)), + "method" => { + let index = DriveMethod::VARIANTS + .iter() + .position(|method| *method == self.method) + .unwrap_or(0); + Some(json!(index)) + } + "mode" => { + let index = Mode::VARIANTS + .iter() + .position(|m| *m == self.mode) + .unwrap_or(0); + Some(json!(index)) + } + "frequency_hz" => Some(json!(self.frequency_hz)), + "min_level" => Some(json!(self.min_level)), + "depth_a" => Some(json!(self.depth_a)), + "operating_point" => Some(json!(self.operating_point)), + "v_null_dac" => Some(json!(self.v_null_dac)), + "v_pi_dac" => Some(json!(self.v_pi_dac)), + "detector_geometry" => { + let index = calibration::DetectorGeometry::VARIANTS + .iter() + .position(|geometry| *geometry == self.detector_geometry) + .unwrap_or(0); + Some(json!(index)) + } + "calibration_dir" => Some(json!(self.calibration_dir)), + // Momentary buttons export a monotonic press counter so a press on + // the UI mirror reaches the live worker through the settings + // snapshot (ADR 010). + "calibrate" => Some(self.press_measure.value()), + "calibrate_apply" => Some(self.press_apply.value()), + "protocol_path" => Some(json!(self.protocol_path)), + // The live worker reports the actual run state; the UI mirror + // reports the operator's request so the settings snapshot can + // transport the start to the worker (which owns the device link). + "protocol_run" => Some(json!( + if self.runtime_role == PluginRuntimeRole::LiveWorker { + self.protocol_active() + } else { + self.protocol_requested + } + )), + _ => None, + } + } + + fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + if self.lease.is_some() { + return Err(format!( + "setting '{key}' is locked while automation holds the modulation lease" + )); + } + match key { + "port" => { + self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); + Ok(()) + } + "connect" => { + let requested = value.as_bool().ok_or("connect must be a boolean")?; + self.connect_requested = requested; + if requested { + self.connect(); + } else { + self.disconnect(); + } + Ok(()) + } + "level" => { + self.level = value + .as_i64() + .ok_or("level must be an integer")? + .clamp(0, self.max_level); + if self.min_level > self.level { + self.min_level = self.level; + } + if self.method == DriveMethod::Manual { + self.send_modulation(); + } + Ok(()) + } + "max_level" => { + self.max_level = value + .as_i64() + .ok_or("max_level must be an integer")? + .clamp(0, MAX_DAC_CODE); + // Lowering the ceiling below the manual peak lowers that peak. + if self.level > self.max_level { + self.level = self.max_level; + } + if self.min_level > self.max_level { + self.min_level = self.max_level; + } + self.send_modulation(); + Ok(()) + } + "method" => { + let method_names: Vec = DriveMethod::VARIANTS + .iter() + .map(|method| method.name().to_owned()) + .collect(); + let name = enum_choice(&value, &method_names)?; + let method = DriveMethod::from_name(&name) + .ok_or_else(|| format!("unknown drive method: {name}"))?; + let previous = self.method; + self.method = method; + if let Err(error) = self.validate_drive() { + self.method = previous; + return Err(error); + } + self.last_error = None; + self.send_modulation(); + Ok(()) + } + "mode" => { + let mode_names: Vec = + Mode::VARIANTS.iter().map(|m| m.name().to_owned()).collect(); + let name = enum_choice(&value, &mode_names)?; + let mode = Mode::from_name(&name).ok_or_else(|| format!("unknown mode: {name}"))?; + let previous = self.mode; + self.mode = mode; + if let Err(error) = self.validate_drive() { + self.mode = previous; + return Err(error); + } + self.send_modulation(); + Ok(()) + } + "frequency_hz" => { + let hz = value.as_f64().ok_or("frequency_hz must be a number")?; + self.frequency_hz = hz.clamp(0.01, 2_000.0); + if self.mode.is_periodic() { + self.send_modulation(); + } + Ok(()) + } + "min_level" => { + self.min_level = value + .as_i64() + .ok_or("min_level must be an integer")? + .clamp(0, self.level); + if self.method == DriveMethod::Manual { + self.send_modulation(); + } + Ok(()) + } + "depth_a" => { + let depth_a = value + .as_f64() + .ok_or("depth_a must be a number")? + .clamp(0.01, 6.0); + let previous = self.depth_a; + self.depth_a = depth_a; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.depth_a = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "operating_point" => { + let operating_point = value + .as_f64() + .ok_or("operating_point must be a number")? + .clamp(0.01, 1.0); + let previous = self.operating_point; + self.operating_point = operating_point; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.operating_point = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "v_null_dac" => { + let v_null_dac = value + .as_i64() + .ok_or("v_null_dac must be an integer")? + .clamp(0, MAX_DAC_CODE); + let previous = self.v_null_dac; + self.v_null_dac = v_null_dac; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.v_null_dac = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "v_pi_dac" => { + let v_pi_dac = value + .as_i64() + .ok_or("v_pi_dac must be an integer")? + .clamp(1, MAX_DAC_CODE); + let previous = self.v_pi_dac; + self.v_pi_dac = v_pi_dac; + if self.method == DriveMethod::Calibrated || self.mode.optical_target().is_some() { + if let Err(error) = self.validate_drive() { + self.v_pi_dac = previous; + return Err(error); + } + self.send_modulation(); + } + Ok(()) + } + "detector_geometry" => { + let variants: Vec = calibration::DetectorGeometry::VARIANTS + .iter() + .map(|geometry| geometry.name().to_owned()) + .collect(); + let chosen = enum_choice(&value, &variants)?; + self.detector_geometry = calibration::DetectorGeometry::from_name(&chosen) + .ok_or("unknown detector geometry")?; + // The stored fit was resolved against the old geometry; re-fit + // rather than leave a V_null that is now a quarter wave out. + if let Some(fit) = self.fit.take() { + match calibration::fit_transfer( + &fit.points, + f64::from(self.max_level.clamp(1, MAX_DAC_CODE) as u16), + self.detector_geometry, + ) { + Ok(refitted) => self.fit = Some(refitted), + Err(error) => self.calibration_status = format!("re-fit failed: {error}"), + } + } + Ok(()) + } + "calibration_dir" => { + self.calibration_dir = value + .as_str() + .ok_or("calibration_dir must be a string")? + .to_owned(); + Ok(()) + } + "calibrate" => { + if !self.press_measure.accept(&value) { + return Ok(()); + } + if self.sweep.is_some() { + self.finish_calibration_sweep("sweep stopped".into()); + } else { + self.start_calibration_sweep(); + } + self.shared.bump(); + Ok(()) + } + "calibrate_apply" => { + if !self.press_apply.accept(&value) { + return Ok(()); + } + self.apply_calibration_fit(); + Ok(()) + } + "protocol_path" => { + self.protocol_path = value + .as_str() + .ok_or("protocol_path must be a string")? + .to_owned(); + Ok(()) + } + "protocol_run" => { + let requested = value.as_bool().ok_or("protocol_run must be a boolean")?; + // The host re-applies the full settings snapshot on every + // sync, so only value *transitions* are actions — otherwise a + // finished protocol would silently restart on the next sync. + if requested == self.protocol_requested { + return Ok(()); + } + self.protocol_requested = requested; + if requested { + // Only the live worker owns the device link; the UI mirror + // records the request and the settings snapshot starts the + // protocol on the worker. Failures surface through status + // entries (like `connect`). + if self.runtime_role == PluginRuntimeRole::LiveWorker { + match self.start_protocol() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + } + } else { + self.stop_protocol(); + } + self.shared.bump(); + Ok(()) + } + _ => Err(format!("unknown setting: {key}")), + } + } + + fn status_entries(&self) -> Vec { + let mut entries = Vec::new(); + let state = self.shared.state.lock().expect("device state lock"); + entries.push(StatusEntry::Text(if state.connected { + format!("Modulation: connected ({})", state.firmware) + } else if self.connect_requested { + "Modulation: connecting…".into() + } else { + "Modulation: disconnected".into() + })); + if let Some(code) = state.board_code { + entries.push(StatusEntry::Text(format!( + "Board: code={code} ({})", + state.board_mod + ))); + } + entries.push(StatusEntry::Text(format!( + "Drive: method={}, mode={}", + self.method.name(), + self.mode.name() + ))); + match self.dac_band() { + Ok((lo, hi, hold)) => entries.push(StatusEntry::Text(format!( + "Resolved DAC band: {lo}..{hi} (hold {hold}, {} codes peak-to-peak)", + hi.saturating_sub(lo) + ))), + Err(error) => entries.push(StatusEntry::Text(format!( + "Resolved DAC band invalid: {error}" + ))), + } + if let Some(target) = self.mode.optical_target() { + match self.optical_warp_table(target) { + Ok(_) => { + let drive = self.optical_drive(target); + entries.push(StatusEntry::Text(format!( + "{}: a={:.2}, I_k={:.2}, V_null={}, Vπ={} @ {:.3} Hz", + self.mode.name(), + drive.depth_a, + drive.operating_point, + self.v_null_dac, + self.v_pi_dac, + self.frequency_hz, + ))); + } + Err(error) => { + entries.push(StatusEntry::Text(format!("Optical drive invalid: {error}"))) + } + } + } + if let Some(run) = &self.protocol { + if let Ok(progress) = run.progress.lock() { + entries.push(StatusEntry::Text(if progress.finished { + if progress.stopped { + "Protocol: stopped (last step holds)".into() + } else { + "Protocol: finished (last step holds)".into() + } + } else { + format!( + "Protocol: loop {}/{} step {}/{} — {}", + progress.loop_index, + progress.loops, + progress.step_index, + progress.total_steps, + progress.summary + ) + })); + } + } + if let Some(sweep) = self.sweep.as_ref() { + entries.push(StatusEntry::Text(format!( + "Calibration: sweeping {}/{}", + sweep.index + 1, + sweep.total() + ))); + } else if !self.calibration_status.is_empty() { + entries.push(StatusEntry::Text(format!( + "Calibration: {}", + self.calibration_status + ))); + } + if let Some(fit) = self.fit.as_ref() { + // The reject-port extremum bounds the anchor from below but is not + // the anchor: the residual transmitted floor is not separable here + // (knowledge base `pockels-waveform-linearisation.md` §4.4). + entries.push(StatusEntry::Text(format!( + "Detector at null: {:.3} V — lower bound on the total-power anchor I_tot, \ + not the anchor itself", + fit.detector_volts_at_null() + ))); + for warning in self.fit_warnings() { + entries.push(StatusEntry::Text(format!("Check: {warning}"))); + } + } + if let Some(calibration_id) = self.calibration_id.as_ref() { + entries.push(StatusEntry::Text(format!( + "Calibration in use: {calibration_id}" + ))); + } + if let Some(error) = state.last_error.clone().or_else(|| self.last_error.clone()) { + entries.push(StatusEntry::Text(format!("Error: {error}"))); + } + entries + } + + fn host_views(&self) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![ + HostDatasetDescriptor { + id: STATUS_DATASET_ID.into(), + title: "Laser modulation".into(), + kind: HostDatasetKind::TableV1(self.status_schema()), + empty_message: "Modulation control idle.".into(), + display: None, + relations: Vec::new(), + }, + HostDatasetDescriptor { + id: CURVE_DATASET_ID.into(), + title: "Pockels transfer curve".into(), + kind: HostDatasetKind::Series1dV1, + empty_message: "Set V_null and Vπ, or measure a transfer curve.".into(), + display: None, + relations: Vec::new(), + }, + ], + views: vec![ + HostViewDescriptor { + id: STATUS_VIEW_ID.into(), + title: "Laser modulation".into(), + dataset_id: STATUS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: CURVE_VIEW_ID.into(), + title: "Pockels transfer curve".into(), + dataset_id: CURVE_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::LineSeriesWindow, + }, + ], + actions: Vec::new(), + } + } + + fn host_view_dataset(&self, dataset_id: &str) -> Option> { + match dataset_id { + STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), + CURVE_DATASET_ID => serde_json::to_vec(&self.curve_dataset()).ok(), + _ => None, + } + } + + fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { + match dataset_id { + STATUS_DATASET_ID | CURVE_DATASET_ID => { + self.shared.generation.load(Ordering::Relaxed).max(1) + } + _ => 0, + } + } +} + +impl Drop for StageAModulationPlugin { + fn drop(&mut self) { + self.disconnect(); + } +} + +export_plugin!(StageAModulationPlugin); + +#[cfg(test)] +mod tests { + use super::*; + use augur_plugin_api::{ExecutionContext, ExecutionMode}; + + fn live_execution() -> ExecutionContext { + ExecutionContext { + mode: ExecutionMode::LiveCapture, + effects_allowed: true, + session_id: Some("test".into()), + } + } + + fn service_request( + plugin: &StageAModulationPlugin, + id: u64, + requester: &str, + command: ModulationCommandV1, + revision: Option, + ) -> PluginServiceRequest { + let mut payload = ModulationRequestV1::new( + stage_a_plugin_contract::RequestId(id), + ClientId::from(requester), + command, + ); + payload.target_owner_instance = Some(plugin.owner_instance.clone()); + payload.run_id = Some(RunId::from("run-a")); + payload.lease_id = Some(LeaseId::from("lease-a")); + payload.requested_revision = revision.map(SemanticRevision); + payload.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id: id, + source_plugin_id: requester.into(), + target_plugin_id: PLUGIN_ID_STAGE_A_MODULATION.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + payload: serde_json::to_value(payload).unwrap(), + } + } + + fn live_plugin() -> StageAModulationPlugin { + let mut plugin = StageAModulationPlugin::default(); + plugin.set_runtime_role(PluginRuntimeRole::LiveWorker); + plugin.effects_allowed = true; + plugin + } + + fn wait_until bool>( + plugin: &StageAModulationPlugin, + timeout: Duration, + done: F, + ) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if done(plugin) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("condition not reached within {timeout:?}"); + } + + fn board_code(plugin: &StageAModulationPlugin) -> Option { + plugin.shared.state.lock().unwrap().board_code + } + + /// Connect checkbox → slider change → MOD sent by the device thread → + /// board echoes the code. No process_frame involved anywhere. + #[test] + fn level_change_transfers_without_frames() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + assert_eq!( + plugin.shared.state.lock().unwrap().firmware, + "0.3.0-mock".to_owned() + ); + + plugin.set_setting("level", json!(1234)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1234) + }); + assert!(plugin.shared.state.lock().unwrap().last_error.is_none()); + + plugin.set_setting("connect", json!(false)).unwrap(); + assert!(!plugin.device_connected()); + } + + /// The max cap bounds the slider, and lowering it re-sends a lower level. + #[test] + fn max_level_caps_the_slider() { + let mut plugin = live_plugin(); + plugin.set_setting("max_level", json!(1000)).unwrap(); + plugin.set_setting("level", json!(4095)).unwrap(); + assert_eq!(plugin.level, 1000, "slider clamps to the cap"); + + plugin.set_setting("max_level", json!(500)).unwrap(); + assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); + + let schema = plugin.settings_schema(); + let level_item = schema.sections[0] + .items + .iter() + .find(|item| item.key == "level") + .expect("level setting exists"); + match &level_item.kind { + SettingKind::I64Slider { max, .. } => assert_eq!(*max, 500), + other => panic!("level must stay a slider, got {other:?}"), + } + } + + /// Square drive with min threshold reaches the mock and starts at min; + /// slider to 0 drives the output to 0. + #[test] + fn square_with_min_threshold_round_trips() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + + plugin.set_setting("level", json!(2000)).unwrap(); + plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); + plugin.set_setting("min_level", json!(500)).unwrap(); + plugin.set_setting("mode", json!("SQUARE")).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(500) + }); + assert!(plugin + .shared + .state + .lock() + .unwrap() + .board_mod + .contains("SQUARE 500..2000")); + + plugin.set_setting("mode", json!("CONST")).unwrap(); + plugin.set_setting("level", json!(0)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(0) + }); + plugin.set_setting("connect", json!(false)).unwrap(); + } + + const TEST_PROTOCOL: &str = r#" +loops = 2 + +[[steps]] +duration_s = 0.03 +wave = "SINE" +level = 2000 +min = 100 +frequency_hz = 100.0 + +[[steps]] +duration_s = 0.03 +wave = "CONST" +level = 750 +"#; + + #[test] + fn protocol_parsing_validates_steps() { + let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); + assert_eq!(loops, 2); + assert_eq!(steps.len(), 2); + let encoded = |command: &Command, seq: u32| { + String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") + }; + assert_eq!( + encoded(&steps[0].command, 1), + "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" + ); + assert_eq!( + encoded(&steps[1].command, 2), + "@2 MOD wave=CONST level=750\n" + ); + assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); + + assert!(parse_protocol("loops = 1").is_err(), "steps required"); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), + "periodic steps need a frequency" + ); + assert!( + parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), + "level range enforced" + ); + assert!( + parse_protocol( + "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" + ) + .is_err(), + "min above level rejected" + ); + let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") + .expect("OFF needs no level"); + assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); + } + + /// A protocol against the mock walks every step, holds the last one, and + /// reports finished. + #[test] + fn protocol_runs_to_completion_on_the_mock() { + let dir = std::env::temp_dir().join(format!( + "stage-a-modulation-protocol-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("protocol.toml"); + std::fs::write(&path, TEST_PROTOCOL).unwrap(); + + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + + plugin + .set_setting("protocol_path", json!(path.display().to_string())) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); + + // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. + wait_until(&plugin, Duration::from_secs(3), |p| { + !p.protocol_active() && board_code(p) == Some(750) + }); + assert!(!plugin.protocol_active()); + assert_eq!(board_code(&plugin), Some(750), "last step holds"); + let progress = plugin + .protocol + .as_ref() + .unwrap() + .progress + .lock() + .unwrap() + .clone(); + assert!(progress.finished && !progress.stopped); + assert_eq!((progress.loop_index, progress.step_index), (2, 2)); + + plugin.set_setting("connect", json!(false)).unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn protocol_requires_a_connection() { + let mut plugin = live_plugin(); + plugin + .set_setting("protocol_path", json!("/tmp/x.toml")) + .unwrap(); + plugin.set_setting("protocol_run", json!(true)).unwrap(); + assert!(plugin + .last_error + .as_deref() + .is_some_and(|err| err.contains("connect"))); + assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); + } + + /// The host settings UI exchanges enum values as indices into the + /// schema's variant list (radio buttons send `json!(index)`). + #[test] + fn enum_settings_round_trip_as_indices() { + let mut plugin = live_plugin(); + // Drive method: index 1 = CALIBRATED. + plugin + .set_setting("method", json!(1)) + .expect("method index accepted"); + assert_eq!(plugin.method, DriveMethod::Calibrated); + assert_eq!(plugin.get_setting("method"), Some(json!(1))); + // Mode: index 2 = SQUARE in the schema's variant order. + plugin + .set_setting("mode", json!(2)) + .expect("index accepted"); + assert_eq!(plugin.mode, Mode::Square); + assert_eq!(plugin.get_setting("mode"), Some(json!(2))); + // Port: index 1 = "mock" (variants start with auto, mock). + plugin + .set_setting("port", json!(1)) + .expect("index accepted"); + assert_eq!(plugin.port_hint, "mock"); + assert_eq!(plugin.get_setting("port"), Some(json!(1))); + // Out-of-range indices are visible errors, not silent no-ops. + assert!(plugin.set_setting("mode", json!(99)).is_err()); + assert!(plugin.set_setting("method", json!(99)).is_err()); + // String names keep working (tests, saved configs). + plugin + .set_setting("mode", json!("SINE")) + .expect("name accepted"); + assert_eq!(plugin.mode, Mode::Sine); + plugin + .set_setting("method", json!("MANUAL")) + .expect("method name accepted"); + assert_eq!(plugin.method, DriveMethod::Manual); + } + + #[test] + fn method_switches_only_its_settings_block() { + let mut plugin = live_plugin(); + let keys = |plugin: &StageAModulationPlugin| { + plugin.settings_schema().sections[0] + .items + .iter() + .map(|item| item.key.clone()) + .collect::>() + }; + + let manual = keys(&plugin); + assert_eq!( + &manual[..6], + [ + "port", + "connect", + "max_level", + "method", + "mode", + "frequency_hz" + ] + ); + assert!(manual.iter().any(|key| key == "level")); + assert!(manual.iter().any(|key| key == "min_level")); + assert!(!manual.iter().any(|key| key == "depth_a")); + assert!(!manual.iter().any(|key| key == "operating_point")); + assert!(!manual.iter().any(|key| key == "v_null_dac")); + assert!(!manual.iter().any(|key| key == "v_pi_dac")); + + let schema = plugin.settings_schema(); + let mode = schema.sections[0] + .items + .iter() + .find(|item| item.key == "mode") + .expect("mode setting"); + match &mode.kind { + SettingKind::Enum { variants, .. } => { + assert_eq!(variants.len(), 5, "all modes stay available"); + } + other => panic!("mode must be an enum, got {other:?}"), + } + + plugin.last_error = Some("stale".into()); + plugin.set_setting("method", json!(1)).unwrap(); + assert!( + plugin.last_error.is_none(), + "method change clears stale errors" + ); + let calibrated = keys(&plugin); + assert_eq!( + &calibrated[..6], + [ + "port", + "connect", + "max_level", + "method", + "mode", + "frequency_hz" + ] + ); + assert!(!calibrated.iter().any(|key| key == "level")); + assert!(!calibrated.iter().any(|key| key == "min_level")); + assert!(calibrated.iter().any(|key| key == "depth_a")); + assert!(calibrated.iter().any(|key| key == "operating_point")); + assert!(calibrated.iter().any(|key| key == "v_null_dac")); + assert!(calibrated.iter().any(|key| key == "v_pi_dac")); + } + + #[test] + fn drive_method_resolves_manual_and_calibrated_bands() { + let mut plugin = live_plugin(); + plugin.level = 1_500; + plugin.min_level = 600; + assert_eq!(plugin.dac_band().unwrap(), (600, 1_500, 1_500)); + + plugin.method = DriveMethod::Calibrated; + // The ±a/2 band only exists for modulating modes (Const resolves to a + // pure hold since the full-lobe fix). + plugin.mode = Mode::Sine; + plugin.v_null_dac = 200; + plugin.v_pi_dac = 1_600; + plugin.operating_point = 0.4; + plugin.depth_a = 0.8; + let inversion = plugin.lobe_inversion(); + let expected_lo = inversion + .dac_for_u(plugin.operating_point * (-0.5 * plugin.depth_a).exp()) + .round() as i64; + let expected_hi = inversion + .dac_for_u(plugin.operating_point * (0.5 * plugin.depth_a).exp()) + .round() as i64; + let expected_hold = inversion.dac_for_u(plugin.operating_point).round() as i64; + assert_eq!( + plugin.dac_band().unwrap(), + (expected_lo, expected_hi, expected_hold) + ); + + plugin.max_level = expected_hi - 1; + assert!(plugin + .dac_band() + .unwrap_err() + .contains("exceeds the max limit")); + } + + #[test] + fn manual_optical_drive_is_derived_from_the_slider_band() { + let mut plugin = live_plugin(); + plugin.v_null_dac = 200; + plugin.v_pi_dac = 1_600; + plugin.min_level = 600; + plugin.level = 1_500; + plugin.depth_a = 5.0; + plugin.operating_point = 0.9; + + for target in [ + waveform::OpticalTarget::LogSine, + waveform::OpticalTarget::LinearSine, + ] { + let drive = plugin.optical_drive(target); + let table = plugin + .optical_warp_table(target) + .expect("valid manual band"); + let min = table.iter().copied().min().unwrap(); + let max = table.iter().copied().max().unwrap(); + assert!((i64::from(min) - plugin.min_level).abs() <= 1); + assert!((i64::from(max) - plugin.level).abs() <= 1); + assert_ne!(drive.depth_a, plugin.depth_a); + assert_ne!(drive.operating_point, plugin.operating_point); + } + } + + #[test] + fn every_mode_drives_under_both_methods() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.max_level = MAX_DAC_CODE; + plugin.min_level = 600; + plugin.level = 1_500; + plugin.v_null_dac = 200; + plugin.v_pi_dac = 1_600; + plugin.operating_point = 0.4; + plugin.depth_a = 0.8; + + for method in DriveMethod::VARIANTS { + plugin.method = method; + for mode in Mode::VARIANTS { + plugin.mode = mode; + plugin.shared.state.lock().unwrap().board_mod.clear(); + plugin.send_modulation(); + assert!( + plugin.last_error.is_none(), + "{} {}: {:?}", + method.name(), + mode.name(), + plugin.last_error + ); + let expected_wave = match mode { + Mode::Const => "CONST", + Mode::Sine => "SINE", + Mode::Square => "SQUARE", + Mode::OpticalLogSine | Mode::OpticalLinearSine => "WARP", + }; + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .board_mod + .starts_with(expected_wave) + }); + let board_mod = plugin.shared.state.lock().unwrap().board_mod.clone(); + assert!( + board_mod.starts_with(expected_wave), + "{} {} produced {board_mod}", + method.name(), + mode.name() + ); + } + } + plugin.disconnect(); + } + + /// min_level can never exceed the level. + #[test] + fn min_threshold_is_clamped_to_level() { + let mut plugin = live_plugin(); + plugin.set_setting("level", json!(1000)).unwrap(); + plugin.set_setting("min_level", json!(3000)).unwrap(); + assert_eq!(plugin.min_level, 1000); + plugin.set_setting("level", json!(200)).unwrap(); + assert_eq!(plugin.min_level, 200, "lowering level drags min down"); + } + + #[test] + fn ui_mirror_never_opens_the_command_port() { + let mut plugin = StageAModulationPlugin::default(); + plugin.port_hint = "mock".into(); + plugin.set_setting("connect", json!(true)).unwrap(); + assert!(plugin.link.is_none()); + assert!(!plugin.device_connected()); + assert!(matches!( + plugin + .handle_service_request( + &service_request( + &plugin, + 1, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ), + &live_execution(), + ) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + } + + /// Runs the sweep to completion against the mock board, synthesizing the + /// light the reject-port photodiode *would* report for whatever code the + /// board is actually holding. The fit must then recover the synthetic + /// lobe, which makes this a ground-truth check of the whole loop: + /// commanding, settle gating, point collection, and the fit. + fn run_sweep_against_mock(plugin: &mut StageAModulationPlugin, v_null: f64, v_pi: f64) { + let mut sample_index = 0_u64; + for _ in 0..4_000 { + let Some((code, _)) = plugin.sweep.as_ref().and_then(CalibrationSweep::current) else { + break; + }; + // Only report light once the board actually holds the commanded + // code. On the bench the settle margin covers the serial + // round-trip; here it is asserted, so a level can never be + // attributed to a code the board had not reached. + if plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.commanded_at_sample.is_some()) + { + wait_until(plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(i64::from(code)) + }); + } + let held = board_code(plugin).unwrap_or(0) as f64; + let u = (std::f64::consts::PI * (held - v_null) / (2.0 * v_pi)) + .sin() + .powi(2); + sample_index += SETTLE_SAMPLES; + plugin.drive_calibration(Some(PhotodiodeLevelV1 { + // Reject port: brightest at the excitation null. + mean_volts: 2.4 - 2.2 * u, + peak_to_peak_volts: 0.001, + sample_count: SETTLE_SAMPLES, + end_sample_index: sample_index, + clipped: false, + })); + } + } + + #[test] + fn sweep_recovers_a_synthetic_lobe_and_restores_the_armed_drive() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + // Arm a drive the sweep must put back afterwards. + plugin.set_setting("level", json!(1_234)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1_234) + }); + + plugin.set_setting("calibrate", json!(true)).unwrap(); + assert!(plugin.sweep.is_some(), "sweep started"); + run_sweep_against_mock(&mut plugin, 300.0, 1_600.0); + + assert!(plugin.sweep.is_none(), "sweep ran to completion"); + let fit = plugin.fit.as_ref().expect("produced a fit"); + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null {}", + fit.v_null_dac + ); + assert!((fit.v_pi_dac - 1_600.0).abs() < 10.0, "Vπ {}", fit.v_pi_dac); + assert_eq!(fit.points.len(), SWEEP_POINTS_PER_PASS * 2); + + // The armed drive is back on the board: a calibration sweep must leave + // the bench as it found it. + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(1_234) + }); + + // Applying writes the lobe through and publishes a calibration id. + assert!( + plugin.fit_warnings().is_empty(), + "{:?}", + plugin.fit_warnings() + ); + plugin.set_setting("calibrate_apply", json!(true)).unwrap(); + assert_eq!(plugin.v_null_dac, 300); + assert!((plugin.v_pi_dac - 1_600).abs() <= 10); + assert!(plugin.calibration_id.is_some()); + assert!(plugin.control_state().calibration_id.is_some()); + } + + /// The host re-applies the **whole** settings snapshot on every sync, and + /// most drive handlers push to the board unconditionally. Without a guard + /// each sync re-arms the operator's waveform on top of the code the sweep + /// just commanded, so every point measures the armed drive instead of the + /// staircase and the fit sees a flat curve. + #[test] + fn a_settings_sync_during_a_sweep_does_not_re_arm_the_operator_drive() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + // Arm a periodic drive, as an operator would before calibrating. + let sine = Mode::VARIANTS + .iter() + .position(|m| *m == Mode::Sine) + .unwrap(); + plugin.set_setting("level", json!(3_000)).unwrap(); + plugin.set_setting("min_level", json!(1_000)).unwrap(); + plugin.set_setting("mode", json!(sine)).unwrap(); + // Let the device thread drain the armed drive, so anything still queued + // below is something the sync put there. + wait_until(&plugin, Duration::from_secs(2), |p| { + p.shared.pending.lock().unwrap().is_none() + }); + + plugin.set_setting("calibrate", json!(true)).unwrap(); + assert!(plugin.sweep.is_some()); + + // Exactly what `apply_live_plugin_snapshot` does: write every key back. + let resync = |plugin: &mut StageAModulationPlugin| { + for key in [ + "frequency_hz", + "level", + "max_level", + "method", + "min_level", + "mode", + "v_null_dac", + "v_pi_dac", + ] { + let value = plugin.get_setting(key).expect("exported"); + plugin.set_setting(key, value).expect("re-applies"); + } + }; + resync(&mut plugin); + assert!( + plugin.shared.pending.lock().unwrap().is_none(), + "a settings sync queued a drive while the sweep owned the DAC" + ); + + // With the sync fighting it on every tick, the sweep must still see the + // staircase and produce a usable fit. + let mut sample_index = 0_u64; + for _ in 0..4_000 { + let Some((code, _)) = plugin.sweep.as_ref().and_then(CalibrationSweep::current) else { + break; + }; + resync(&mut plugin); + if plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.commanded_at_sample.is_some()) + { + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p) == Some(i64::from(code)) + }); + } + let held = board_code(&plugin).unwrap_or(0) as f64; + let u = (std::f64::consts::PI * (held - 300.0) / 3_200.0) + .sin() + .powi(2); + sample_index += SETTLE_SAMPLES; + plugin.drive_calibration(Some(PhotodiodeLevelV1 { + mean_volts: 2.4 - 2.2 * u, + peak_to_peak_volts: 0.001, + sample_count: SETTLE_SAMPLES, + end_sample_index: sample_index, + clipped: false, + })); + } + + let fit = plugin + .fit + .as_ref() + .unwrap_or_else(|| panic!("no fit: {}", plugin.calibration_status)); + assert!( + (fit.v_null_dac - 300.0).abs() < 5.0, + "V_null {}", + fit.v_null_dac + ); + + // The armed sine comes back once the sweep releases the DAC. + wait_until(&plugin, Duration::from_secs(2), |p| { + board_code(p).is_some_and(|code| code != 0) + }); + assert_eq!(plugin.mode, Mode::Sine); + } + + #[test] + fn sweep_waits_for_a_window_measured_after_the_code_was_commanded() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + plugin.set_setting("calibrate", json!(true)).unwrap(); + + let stale = |end_sample_index| PhotodiodeLevelV1 { + mean_volts: 1.0, + peak_to_peak_volts: 0.001, + sample_count: 100, + end_sample_index, + clipped: false, + }; + // First tick commands the point and adopts the sample index. + plugin.drive_calibration(Some(stale(10_000))); + assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 0); + // A window that began before the command must not be accepted, however + // many times it arrives — this is what makes settling provable. + for _ in 0..5 { + plugin.drive_calibration(Some(stale(10_050))); + } + assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 0); + // Once the window starts past the settle margin the point is taken. + plugin.drive_calibration(Some(stale(10_000 + SETTLE_SAMPLES + 100))); + assert_eq!(plugin.sweep.as_ref().unwrap().points.len(), 1); + } + + #[test] + fn sweep_is_refused_while_automation_holds_the_lease() { + let mut plugin = live_plugin(); + plugin.set_setting("port", json!("mock")).unwrap(); + plugin.set_setting("connect", json!(true)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + plugin.handle_service_request( + &service_request( + &plugin, + 1, + "stage-a-a1", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ), + &live_execution(), + ); + assert!(plugin.lease.is_some()); + // Two owners stepping the same DAC would interleave silently. + plugin.start_calibration_sweep(); + assert!(plugin.sweep.is_none()); + assert!( + plugin.calibration_status.contains("leased"), + "{}", + plugin.calibration_status + ); + } + + fn synthetic_fit( + span: f64, + max_code: u16, + edit: impl Fn(&mut calibration::SweepPoint, usize), + ) -> calibration::TransferFit { + let points: Vec = calibration::sweep_codes(max_code, 49, false) + .into_iter() + .enumerate() + .map(|(index, (code, direction))| { + let u = (std::f64::consts::PI * (f64::from(code) - 300.0) / 1_720.0) + .sin() + .powi(2); + let mut point = calibration::SweepPoint { + code, + direction, + volts: 0.098 + span * u, + peak_to_peak_volts: 0.001, + clipped: false, + }; + edit(&mut point, index); + point + }) + .collect(); + calibration::fit_transfer( + &points, + f64::from(max_code), + calibration::DetectorGeometry::RejectedComplement, + ) + .expect("fits") + } + + /// A stray sample inflates the RMS residual several fold while leaving the + /// fitted period accurate. Dropping the wild points keeps the reported + /// residual describing the curve instead of the worst sample. + #[test] + fn a_stray_point_is_dropped_instead_of_ruining_the_fit() { + let clean = synthetic_fit(-0.090, 4_095, |_, _| {}); + let strayed = synthetic_fit(-0.090, 4_095, |point, index| { + if index == 20 { + point.volts += 0.09; + } + }); + + assert_eq!(clean.rejected_points, 0); + assert_eq!(strayed.rejected_points, 1, "the stray should be dropped"); + assert!( + (strayed.v_pi_dac - clean.v_pi_dac).abs() < 5.0, + "Vpi moved from {} to {}", + clean.v_pi_dac, + strayed.v_pi_dac + ); + assert!( + strayed.quality < 0.02, + "residual still dominated by the stray: {:.1}%", + strayed.quality * 100.0 + ); + // The plot still shows every measured point, stray included. + assert_eq!(strayed.points.len(), 49); + } + + /// A poor residual is the operator's call, made against the plot — it warns + /// but never blocks, because a stray sample can inflate it while the fitted + /// lobe stays good. The one genuinely meaningless case, a lobe that does not + /// fit inside the commandable range, is refused by the fit itself. + #[test] + fn a_scattered_or_clipped_fit_warns_but_still_applies() { + let mut plugin = live_plugin(); + plugin.fit = Some(synthetic_fit(-0.090, 4_095, |point, index| { + point.clipped = point.code < 100; + if index % 7 == 0 { + point.volts += 0.004; + } + })); + + let warnings = plugin.fit_warnings().join(" | "); + assert!(warnings.contains("clipped"), "{warnings}"); + + plugin.set_setting("calibrate_apply", json!(true)).unwrap(); + assert!( + plugin.calibration_id.is_some(), + "{}", + plugin.calibration_status + ); + assert!( + (plugin.v_pi_dac - 860).abs() <= 10, + "Vpi {}", + plugin.v_pi_dac + ); + } + + fn calibration_button(plugin: &StageAModulationPlugin, key: &str) -> SettingKind { + plugin + .settings_schema() + .sections + .iter() + .flat_map(|section| section.items.iter()) + .find(|item| item.key == key) + .unwrap_or_else(|| panic!("{key} is in the schema")) + .kind + .clone() + } + + /// The UI mirror renders the settings schema, and it never owns the device + /// link, a lease, a sweep, or a fit. Gating `enabled` on any of those + /// disables the buttons permanently — the operator can never start. + #[test] + fn calibration_buttons_are_offered_on_the_ui_mirror() { + let mut mirror = StageAModulationPlugin::default(); + assert_eq!(mirror.runtime_role, PluginRuntimeRole::UiMirror); + mirror.port_hint = "mock".into(); + + for key in ["calibrate", "calibrate_apply"] { + assert!( + matches!( + calibration_button(&mirror, key), + SettingKind::Button { enabled: false } + ), + "{key} should be off before the operator asks to connect" + ); + } + + mirror.set_setting("connect", json!(true)).unwrap(); + // The mirror deliberately never opens the port... + assert!(mirror.link.is_none()); + // ...but the buttons must still be pressable, because the worker — not + // the mirror — owns the link and enforces the real interlocks. + for key in ["calibrate", "calibrate_apply"] { + assert!( + matches!( + calibration_button(&mirror, key), + SettingKind::Button { enabled: true } + ), + "{key} is disabled on the mirror, so it can never be pressed" + ); + } + } + + /// A press is transported mirror → worker as a monotonic counter. The + /// worker adopts the counter it first sees as a baseline so a reload does + /// not replay old presses — but that baseline must not swallow the + /// operator's first real press. + #[test] + fn a_forwarded_press_reaches_a_freshly_loaded_worker() { + let mut mirror = StageAModulationPlugin::default(); + let mut worker = live_plugin(); + worker.port_hint = "mock".into(); + worker.set_setting("connect", json!(true)).unwrap(); + wait_until(&worker, Duration::from_secs(2), |p| p.device_connected()); + + // The host syncs the settings snapshot before anything is clicked. + let sync = |worker: &mut StageAModulationPlugin, mirror: &StageAModulationPlugin| { + let value = mirror.get_setting("calibrate").expect("exported"); + worker.set_setting("calibrate", value).unwrap(); + }; + sync(&mut worker, &mirror); + assert!( + worker.sweep.is_none(), + "a plain sync must not start a sweep" + ); + + // First real click on the mirror, then the next settings sync. + mirror.set_setting("calibrate", json!(true)).unwrap(); + sync(&mut worker, &mirror); + assert!( + worker.sweep.is_some(), + "the operator's first press never reached the worker" + ); + + // Re-syncing the same counter must not re-trigger. + sync(&mut worker, &mirror); + assert!(worker.sweep.is_some()); + // A second click stops it, proving the toggle survives the transport. + mirror.set_setting("calibrate", json!(true)).unwrap(); + sync(&mut worker, &mirror); + assert!( + worker.sweep.is_none(), + "second press should abort the sweep" + ); } - fn board_code(plugin: &StageAModulationPlugin) -> Option { - plugin.shared.state.lock().unwrap().board_code + #[test] + fn the_curve_view_shows_the_configured_lobe_before_any_measurement() { + let mut plugin = live_plugin(); + plugin.set_setting("v_null_dac", json!(400)).unwrap(); + plugin.set_setting("v_pi_dac", json!(900)).unwrap(); + let curve = plugin.curve_dataset(); + // Normalised until something has actually been measured. + assert!(curve.y_label.contains("normalised")); + let names: Vec<&str> = curve.lines.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["configured lobe", "V_null", "V_null + Vπ"]); + let lobe = &curve.lines[0].points; + // Minimum at V_null, maximum a quarter wave later. + let at = |code: f64| { + lobe.iter() + .min_by(|a, b| (a.x - code).abs().total_cmp(&(b.x - code).abs())) + .expect("sampled") + .y + }; + assert!(at(400.0) < 0.01, "u at V_null = {}", at(400.0)); + assert!(at(1_300.0) > 0.99, "u at V_null+Vπ = {}", at(1_300.0)); } - /// Connect checkbox → slider change → MOD sent by the device thread → - /// board echoes the code. No process_frame involved anywhere. #[test] - fn level_change_transfers_without_frames() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); - assert_eq!( - plugin.shared.state.lock().unwrap().firmware, - "0.3.0-mock".to_owned() - ); + fn calibrated_const_hold_spans_the_full_lobe_without_a_headroom() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Const; + plugin.v_null_dac = 1_630; + plugin.v_pi_dac = 860; + plugin.depth_a = 0.5; // must be irrelevant for a constant hold - plugin.set_setting("level", json!(1234)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| { - board_code(p) == Some(1234) - }); - assert!(plugin.shared.state.lock().unwrap().last_error.is_none()); + // I_k = 1 holds exactly at V_null + Vπ (previously rejected because + // the modulated band u_k·e^{a/2} > 1 was demanded even for CONST). + plugin.operating_point = 1.0; + let (lo, hi, hold) = plugin.dac_band().expect("full-lobe hold"); + assert_eq!((lo, hi, hold), (2_490, 2_490, 2_490)); - plugin.set_setting("connect", json!(false)).unwrap(); - assert!(!plugin.device_connected()); + // The user's measured low point: dac_for_u(0.01) ≈ 1685. + plugin.operating_point = 0.01; + let (_, _, hold) = plugin.dac_band().expect("low hold"); + assert_eq!(hold, 1_685); + + // Modulating modes still require the ±a/2 headroom. + plugin.mode = Mode::Sine; + plugin.operating_point = 1.0; + assert!(plugin.dac_band().is_err()); } - /// The max cap bounds the slider, and lowering it re-sends a lower level. #[test] - fn max_level_caps_the_slider() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("max_level", json!(1000)).unwrap(); - plugin.set_setting("level", json!(4095)).unwrap(); - assert_eq!(plugin.level, 1000, "slider clamps to the cap"); + fn rejected_operating_point_does_not_diverge_from_the_board_target() { + let mut plugin = live_plugin(); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Sine; + plugin.v_null_dac = 1_630; + plugin.v_pi_dac = 860; + plugin.depth_a = 0.5; + plugin.operating_point = 0.5; - plugin.set_setting("max_level", json!(500)).unwrap(); - assert_eq!(plugin.level, 500, "lowering the cap lowers the level"); + let error = plugin + .set_setting("operating_point", json!(1.0)) + .expect_err("periodic I_k=1 has no modulation headroom"); + assert!(error.contains("lobe ceiling")); + assert_eq!(plugin.operating_point, 0.5); - let schema = plugin.settings_schema(); - let level_item = schema.sections[0] - .items - .iter() - .find(|item| item.key == "level") - .expect("level setting exists"); - match &level_item.kind { - SettingKind::I64Slider { max, .. } => assert_eq!(*max, 500), - other => panic!("level must stay a slider, got {other:?}"), - } + plugin.set_setting("mode", json!(0)).expect("CONST"); + plugin + .set_setting("operating_point", json!(1.0)) + .expect("CONST maps I_k directly"); + assert_eq!(plugin.dac_band().unwrap(), (2_490, 2_490, 2_490)); } - /// Square drive with min threshold reaches the mock and starts at min; - /// slider to 0 drives the output to 0. #[test] - fn square_with_min_threshold_round_trips() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + fn calibrated_const_sends_the_expected_codes_to_the_board() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Const; + plugin.v_null_dac = 1_630; + plugin.v_pi_dac = 860; - plugin.set_setting("level", json!(2000)).unwrap(); - plugin.set_setting("frequency_hz", json!(10.0)).unwrap(); - plugin.set_setting("min_level", json!(500)).unwrap(); - plugin.set_setting("mode", json!("SQUARE")).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| { - board_code(p) == Some(500) + plugin + .set_setting("operating_point", json!(1.0)) + .expect("full lobe"); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.shared.state.lock().unwrap().board_code == Some(2_490) }); - assert!(plugin - .shared - .state - .lock() - .unwrap() - .board_mod - .contains("SQUARE 500..2000")); - plugin.set_setting("mode", json!("CONST")).unwrap(); - plugin.set_setting("level", json!(0)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| { - board_code(p) == Some(0) + plugin + .set_setting("operating_point", json!(0.01)) + .expect("low point"); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.shared.state.lock().unwrap().board_code == Some(1_685) }); - plugin.set_setting("connect", json!(false)).unwrap(); + plugin.disconnect(); } - const TEST_PROTOCOL: &str = r#" -loops = 2 - -[[steps]] -duration_s = 0.03 -wave = "SINE" -level = 2000 -min = 100 -frequency_hz = 100.0 - -[[steps]] -duration_s = 0.03 -wave = "CONST" -level = 750 -"#; + #[test] + fn ui_armed_drive_publishes_a_board_echo_acknowledged_target() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + // Arm a sine purely through the operator settings — no lease, no + // service request. Consumers (A1) must still see the frequency. + plugin.set_setting("mode", json!(1)).unwrap(); // Sine + plugin.set_setting("frequency_hz", json!(5.0)).unwrap(); + plugin.set_setting("level", json!(1_000)).unwrap(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .board_mod + .starts_with("SINE") + }); + let snapshot = plugin.control_state(); + let target = snapshot.acknowledged.expect("board-echo target"); + assert_eq!(target.revision, SemanticRevision(0)); + match target.waveform.expect("waveform") { + WaveformV1::Periodic { + frequency_millihz, .. + } => assert_eq!(frequency_millihz, 5_000), + other => panic!("expected periodic waveform, got {other:?}"), + } + plugin.disconnect(); + } #[test] - fn protocol_parsing_validates_steps() { - let (steps, loops) = parse_protocol(TEST_PROTOCOL).expect("valid protocol"); - assert_eq!(loops, 2); - assert_eq!(steps.len(), 2); - let encoded = |command: &Command, seq: u32| { - String::from_utf8(command.encode(seq).expect("encodes")).expect("utf8") - }; - assert_eq!( - encoded(&steps[0].command, 1), - "@1 MOD wave=SINE level=2000 min=100 freq_mhz=100000\n" - ); - assert_eq!( - encoded(&steps[1].command, 2), - "@2 MOD wave=CONST level=750\n" + fn set_optical_depth_requires_lease_and_a_calibrated_drive() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Sine; + + // Without a lease the retarget is refused. + let unleased = service_request( + &plugin, + 30, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_250, + }, + None, ); - assert!((steps[0].duration.as_secs_f64() - 0.03).abs() < 1e-9); + assert!(matches!( + plugin + .handle_service_request(&unleased, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); - assert!(parse_protocol("loops = 1").is_err(), "steps required"); - assert!( - parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100").is_err(), - "periodic steps need a frequency" + let acquire = service_request( + &plugin, + 31, + "stage-a-a1", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, ); - assert!( - parse_protocol("[[steps]]\nduration_s = 1.0\nwave = \"CONST\"\nlevel = 9999").is_err(), - "level range enforced" + plugin.handle_service_request(&acquire, &live_execution()); + + let retarget = service_request( + &plugin, + 32, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_250, + }, + None, ); - assert!( - parse_protocol( - "[[steps]]\nduration_s = 1.0\nwave = \"SINE\"\nlevel = 100\nmin = 200\nfrequency_hz = 10.0" - ) - .is_err(), - "min above level rejected" + assert!(matches!( + plugin + .handle_service_request(&retarget, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + )); + assert!((plugin.depth_a - 1.25).abs() < 1e-9); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .board_mod + .starts_with("SINE") + }); + + // The manual DAC band cannot express an optical depth. + plugin.method = DriveMethod::Manual; + let manual = service_request( + &plugin, + 33, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_000, + }, + None, ); - let (off, _) = parse_protocol("[[steps]]\nduration_s = 0.5\nwave = \"OFF\"") - .expect("OFF needs no level"); - assert_eq!(encoded(&off[0].command, 1), "@1 MOD wave=OFF\n"); + assert!(matches!( + plugin + .handle_service_request(&manual, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + plugin.disconnect(); } - /// A protocol against the mock walks every step, holds the last one, and - /// reports finished. #[test] - fn protocol_runs_to_completion_on_the_mock() { - let dir = std::env::temp_dir().join(format!( - "stage-a-modulation-protocol-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() + fn lease_acquire_is_idempotent_and_exclusive_without_frames() { + let mut plugin = live_plugin(); + let acquire = service_request( + &plugin, + 10, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + let first = plugin.handle_service_request(&acquire, &live_execution()); + let expiry = plugin.lease.as_ref().unwrap().expires_at_unix_ms; + let duplicate = plugin.handle_service_request(&acquire, &live_execution()); + assert_eq!(first, duplicate); + assert_eq!(plugin.lease.as_ref().unwrap().expires_at_unix_ms, expiry); + assert!(plugin.set_setting("level", json!(1)).is_err()); + + let conflict = service_request( + &plugin, + 11, + "workflow-b", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&conflict, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } )); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("protocol.toml"); - std::fs::write(&path, TEST_PROTOCOL).unwrap(); + } - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("port", json!("mock")).unwrap(); - plugin.set_setting("connect", json!(true)).unwrap(); - wait_until(&plugin, Duration::from_secs(2), |p| p.device_connected()); + #[test] + fn prepare_safe_off_and_release_publish_terminal_ack_before_lease_loss() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); - plugin - .set_setting("protocol_path", json!(path.display().to_string())) - .unwrap(); - plugin.set_setting("protocol_run", json!(true)).unwrap(); - assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); - assert_eq!(plugin.get_setting("protocol_run"), Some(json!(true))); + let acquire = service_request( + &plugin, + 20, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + let prepare = service_request( + &plugin, + 21, + "workflow-a", + ModulationCommandV1::PrepareA1 { + configuration: A1AcquisitionConfigV1 { + waveform: stage_a_plugin_contract::PeriodicWaveformV1::Sine, + frequency_millihz: 10_000, + center_dac: 1_000, + amplitude_dac: 250, + sample_rate_hz: 20_000, + block_samples: 256, + emit_raw_samples: true, + emit_summary: true, + optical_lut_id: None, + }, + }, + Some(1), + ); + let initial = plugin.handle_service_request(&prepare, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = initial.outcome else { + panic!("prepare rejected"); + }; + let response: ModulationResponseV1 = serde_json::from_value(payload).unwrap(); + assert_eq!(response.common.outcome, RequestOutcomeV1::InProgress); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .last_response + .as_ref() + .is_some_and(|response| { + response.common.request_id.0 == 21 + && response.common.outcome == RequestOutcomeV1::Applied + }) + }); + let terminal = plugin.handle_service_request(&prepare, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = terminal.outcome else { + panic!("terminal prepare rejected"); + }; + let response: ModulationResponseV1 = serde_json::from_value(payload).unwrap(); + assert_eq!( + response.common.acknowledged_revision, + Some(SemanticRevision(1)) + ); - // 2 loops × 2 steps × 30 ms ≈ 120 ms; wait for the final CONST 750. - wait_until(&plugin, Duration::from_secs(3), |p| { - !p.protocol_active() && board_code(p) == Some(750) + let safe_off = service_request( + &plugin, + 22, + "workflow-a", + ModulationCommandV1::SafeOff { + reason: "test".into(), + }, + Some(2), + ); + plugin.handle_service_request(&safe_off, &live_execution()); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .acknowledged + .as_ref() + .is_some_and(|target| { + target.revision == SemanticRevision(2) + && target.waveform == Some(WaveformV1::Off) + }) }); - assert!(!plugin.protocol_active()); - assert_eq!(board_code(&plugin), Some(750), "last step holds"); - let progress = plugin - .protocol - .as_ref() - .unwrap() - .progress - .lock() - .unwrap() - .clone(); - assert!(progress.finished && !progress.stopped); - assert_eq!((progress.loop_index, progress.step_index), (2, 2)); - plugin.set_setting("connect", json!(false)).unwrap(); - std::fs::remove_dir_all(dir).unwrap(); + let release = service_request( + &plugin, + 23, + "workflow-a", + ModulationCommandV1::ReleaseLease { + safe_off: true, + reason: "done".into(), + }, + Some(3), + ); + plugin.handle_service_request(&release, &live_execution()); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner + .shared + .state + .lock() + .unwrap() + .last_response + .as_ref() + .is_some_and(|response| { + response.common.request_id.0 == 23 + && response.common.outcome == RequestOutcomeV1::Applied + }) + }); + plugin.apply_execution_context(&live_execution()); + let snapshot = plugin.control_state(); + assert!( + snapshot.lease.is_some(), + "terminal ACK snapshot retains lease" + ); + let duplicate = plugin.handle_service_request(&release, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = duplicate.outcome else { + panic!("release duplicate rejected"); + }; + let response: ModulationResponseV1 = serde_json::from_value(payload).unwrap(); + assert_eq!(response.common.outcome, RequestOutcomeV1::Applied); + plugin.apply_execution_context(&live_execution()); + assert!(plugin.lease.is_none(), "lease clears after ACK publication"); + plugin.disconnect(); } #[test] - fn protocol_requires_a_connection() { - let mut plugin = StageAModulationPlugin::default(); - plugin - .set_setting("protocol_path", json!("/tmp/x.toml")) - .unwrap(); - plugin.set_setting("protocol_run", json!(true)).unwrap(); + fn lease_expiry_and_effect_revocation_fail_closed_without_frames() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + let acquire = service_request( + &plugin, + 30, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + plugin.lease.as_mut().unwrap().expires_at_unix_ms = now_unix_ms().saturating_sub(1); + plugin.apply_execution_context(&live_execution()); + assert!(plugin.lease.is_none()); assert!(plugin .last_error .as_deref() - .is_some_and(|err| err.contains("connect"))); - assert_eq!(plugin.get_setting("protocol_run"), Some(json!(false))); - } - - /// The host settings UI exchanges enum values as indices into the - /// schema's variant list (radio buttons send `json!(index)`). - #[test] - fn enum_settings_round_trip_as_indices() { - let mut plugin = StageAModulationPlugin::default(); - // Mode: index 2 = SQUARE in the schema's variant order. - plugin - .set_setting("mode", json!(2)) - .expect("index accepted"); - assert_eq!(plugin.mode, Mode::Square); - assert_eq!(plugin.get_setting("mode"), Some(json!(2))); - // Port: index 1 = "mock" (variants start with auto, mock). - plugin - .set_setting("port", json!(1)) - .expect("index accepted"); - assert_eq!(plugin.port_hint, "mock"); - assert_eq!(plugin.get_setting("port"), Some(json!(1))); - // Out-of-range indices are visible errors, not silent no-ops. - assert!(plugin.set_setting("mode", json!(99)).is_err()); - // String names keep working (tests, saved configs). - plugin - .set_setting("mode", json!("SINE")) - .expect("name accepted"); - assert_eq!(plugin.mode, Mode::Sine); - } + .is_some_and(|message| message.contains("lease expired"))); - /// min_level can never exceed the level. - #[test] - fn min_threshold_is_clamped_to_level() { - let mut plugin = StageAModulationPlugin::default(); - plugin.set_setting("level", json!(1000)).unwrap(); - plugin.set_setting("min_level", json!(3000)).unwrap(); - assert_eq!(plugin.min_level, 1000); - plugin.set_setting("level", json!(200)).unwrap(); - assert_eq!(plugin.min_level, 200, "lowering level drags min down"); + let acquire = service_request( + &plugin, + 31, + "workflow-a", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + assert!(plugin.lease.is_some()); + plugin.apply_execution_context(&ExecutionContext::fail_closed()); + assert!(plugin.link.is_none()); + assert!(plugin.lease.is_none()); } } diff --git a/plugins/stage-a-modulation/src/waveform.rs b/plugins/stage-a-modulation/src/waveform.rs new file mode 100644 index 0000000..0485f19 --- /dev/null +++ b/plugins/stage-a-modulation/src/waveform.rs @@ -0,0 +1,389 @@ +//! Optical-target DAC warp-table synthesis for the Pockels/PBS modulator. +//! +//! On one monotonic Pockels/PBS lobe the excitation transfer is +//! `I(V) = I_floor + (I_ceil - I_floor) · sin²(α (V - V_null))`, with +//! `α = π / (2 Vπ)`. The manufacturer likewise describes the amplitude +//! modulator as `sin²`; a 50 % bias only *approximately* linearises the small +//! signal. A pure DAC sine therefore does **not** produce a sinusoidal optical +//! target — it must be pre-warped by inverting the transfer: +//! +//! ```text +//! u(t) = (I_d(t) - I_floor) / (I_ceil - I_floor) // normalised target +//! V(u) = V_null + (2 Vπ / π) · arcsin(√u) // increasing lobe +//! ``` +//! +//! Two optical targets are supported (the drive picks one): +//! - [`OpticalTarget::LogSine`] — `ln I_d = ln I_g + (a/2) sin ωt`, the clean A1 +//! input because the event camera responds to changes in `ln I`. +//! - [`OpticalTarget::LinearSine`] — `I_d = I_c (1 + m sin ωt)`, `m = tanh(a/2)`. +//! +//! The inversion parameters `V_null` and `Vπ` are expressed in **DAC codes** and +//! are settable: the engineer should not rely on nominal `Vπ` but sweep settled +//! constant DAC codes, measure the actual optical transfer, and enter the frozen +//! `V_null` / `Vπ` of one monotonic lobe. A fully measured lookup table can +//! replace this analytic inversion later behind the same interface. + +use std::f64::consts::PI; + +/// Warp-table length played back over one modulation period. +pub const WARP_TABLE_LEN: usize = 256; +/// Full-scale DAC code (12-bit). +pub const DAC_FULL_SCALE: u16 = 4_095; + +/// Optical intensity target the drive should reproduce, swung around the +/// operating point `u_k`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OpticalTarget { + /// Recommended A1 log-intensity sine: `ln I = ln I_k + (a/2) sin ωt`. + LogSine, + /// Literal linear-intensity sine: `I = I_k (1 + m sin ωt)`, `m = tanh(a/2)`. + LinearSine, +} + +/// Frozen inversion of one monotonic Pockels/PBS lobe, in DAC codes. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct LobeInversion { + /// DAC code where the excitation light is at its minimum (`sin² = 0`). + pub v_null_dac: f64, + /// DAC-code distance from `v_null` to the excitation maximum (quarter wave). + pub v_pi_dac: f64, +} + +impl LobeInversion { + /// Normalised optical intensity produced by `code` on the configured lobe: + /// `u = sin²(π(code - V_null) / (2 Vπ))`. + pub fn u_for_dac(&self, code: f64) -> f64 { + let alpha = PI / (2.0 * self.v_pi_dac); + (alpha * (code - self.v_null_dac)).sin().powi(2) + } + + /// DAC code producing normalised optical intensity `u ∈ [0, 1]` on the + /// increasing lobe. + pub fn dac_for_u(&self, u: f64) -> f64 { + self.v_null_dac + (2.0 * self.v_pi_dac / PI) * u.clamp(0.0, 1.0).sqrt().asin() + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct OpticalDrive { + pub target: OpticalTarget, + /// Optical log-modulation depth `a = ln(I_max / I_min)`, must be positive. + pub depth_a: f64, + /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0, 1]`: + /// the geometric-mean point the modulation swings around. Held fixed while + /// `a` is swept, so one response curve keeps `I_k` constant. + pub operating_point: f64, + pub inversion: LobeInversion, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum WarpError { + /// `a` is not finite or not positive. + InvalidDepth, + /// The operating point is not in `(0, 1]`. + InvalidOperatingPoint, + /// `Vπ` is not finite or not positive. + InvalidInversion, + /// The peak optical target exceeds the lobe ceiling (`u_k · peak > 1`): the + /// operating point is too bright for this depth and would saturate. + Saturates { peak: f64 }, + /// A computed DAC code falls outside `0..=4095`: the inversion parameters do + /// not fit the requested depth on this lobe. Clamping would silently distort + /// the optical target, so the drive is refused instead. + OutOfRange { index: usize, code: f64 }, +} + +impl std::fmt::Display for WarpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidDepth => f.write_str("optical depth a must be finite and positive"), + Self::InvalidOperatingPoint => f.write_str("operating point must be in (0, 1]"), + Self::InvalidInversion => f.write_str("Vπ must be finite and positive"), + Self::Saturates { peak } => write!( + f, + "peak optical target u = {peak:.3} exceeds the lobe ceiling; lower a or the operating point" + ), + Self::OutOfRange { index, code } => write!( + f, + "warp sample {index} = {code:.1} DAC leaves 0..=4095; reduce a or re-measure the lobe" + ), + } + } +} + +impl std::error::Error for WarpError {} + +impl OpticalDrive { + /// Derives the target-law parameters that make an optical waveform span + /// the intensities produced by the supplied DAC band. + pub fn from_dac_band( + target: OpticalTarget, + inversion: LobeInversion, + lo: f64, + hi: f64, + ) -> Self { + let u_lo = inversion.u_for_dac(lo); + let u_hi = inversion.u_for_dac(hi); + let (operating_point, depth_a) = match target { + OpticalTarget::LogSine => ((u_lo * u_hi).sqrt(), (u_hi / u_lo).ln()), + OpticalTarget::LinearSine => { + let operating_point = 0.5 * (u_lo + u_hi); + let modulation = (u_hi - u_lo) / (u_hi + u_lo); + (operating_point, 2.0 * modulation.atanh()) + } + }; + Self { + target, + depth_a, + operating_point, + inversion, + } + } + + /// Normalised optical target `u(φ)` for phase fraction `φ ∈ [0, 1)`, swung + /// around the operating point `u_k` (not peak-normalised). + pub fn normalised_intensity(&self, phase: f64) -> f64 { + let sine = (2.0 * PI * phase).sin(); + match self.target { + // ln I = ln I_k + (a/2) sin ωt. + OpticalTarget::LogSine => self.operating_point * (0.5 * self.depth_a * sine).exp(), + // I = I_k (1 + m sin ωt), m = tanh(a/2). + OpticalTarget::LinearSine => { + let m = (0.5 * self.depth_a).tanh(); + self.operating_point * (1.0 + m * sine) + } + } + } + + /// Peak normalised optical target over one period. + fn peak_intensity(&self) -> f64 { + match self.target { + OpticalTarget::LogSine => self.operating_point * (0.5 * self.depth_a).exp(), + OpticalTarget::LinearSine => self.operating_point * (1.0 + (0.5 * self.depth_a).tanh()), + } + } + + /// Builds the `WARP_TABLE_LEN`-entry DAC warp table for one period. + pub fn warp_table(&self) -> Result, WarpError> { + if !self.depth_a.is_finite() || self.depth_a <= 0.0 { + return Err(WarpError::InvalidDepth); + } + if !self.operating_point.is_finite() + || !(0.0..=1.0).contains(&self.operating_point) + || self.operating_point <= 0.0 + { + return Err(WarpError::InvalidOperatingPoint); + } + if !self.inversion.v_pi_dac.is_finite() || self.inversion.v_pi_dac <= 0.0 { + return Err(WarpError::InvalidInversion); + } + let peak = self.peak_intensity(); + if peak > 1.0 + 1e-9 { + return Err(WarpError::Saturates { peak }); + } + let mut table = Vec::with_capacity(WARP_TABLE_LEN); + for index in 0..WARP_TABLE_LEN { + let phase = index as f64 / WARP_TABLE_LEN as f64; + let code = self.inversion.dac_for_u(self.normalised_intensity(phase)); + if !code.is_finite() || code < -0.5 || code > f64::from(DAC_FULL_SCALE) + 0.5 { + return Err(WarpError::OutOfRange { index, code }); + } + table.push(code.round().clamp(0.0, f64::from(DAC_FULL_SCALE)) as u16); + } + Ok(table) + } +} + +/// Forward Pockels/PBS transfer used to verify a warp table reproduces the +/// intended optical target: `u = sin²(α (code - V_null))`, `α = π / (2 Vπ)`. +#[cfg(test)] +pub fn lobe_transmission(code: f64, inversion: &LobeInversion) -> f64 { + inversion.u_for_dac(code) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn inversion() -> LobeInversion { + // Null at code 200, quarter wave 1600 codes later (peak light at 1800). + LobeInversion { + v_null_dac: 200.0, + v_pi_dac: 1_600.0, + } + } + + /// Peak-normalised operating point (max light at the lobe ceiling) so the + /// range/round-trip assertions exercise the full swing. + fn peak_operating_point(target: OpticalTarget, depth_a: f64) -> f64 { + match target { + OpticalTarget::LogSine => (-0.5 * depth_a).exp(), + OpticalTarget::LinearSine => 1.0 / (1.0 + (0.5 * depth_a).tanh()), + } + } + + fn drive(target: OpticalTarget, depth_a: f64) -> OpticalDrive { + OpticalDrive { + target, + depth_a, + operating_point: peak_operating_point(target, depth_a), + inversion: inversion(), + } + } + + #[test] + fn tables_stay_inside_the_dac_range_for_both_targets() { + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + let table = drive(target, 1.0).warp_table().expect("in range"); + assert_eq!(table.len(), WARP_TABLE_LEN); + assert!(table.iter().all(|&code| code <= DAC_FULL_SCALE)); + } + } + + #[test] + fn warp_table_reproduces_the_optical_target_through_the_sin2_transfer() { + // Feeding the warp codes back through the sin² lobe must recover the + // intended normalised intensity: that is the whole point of the warp. + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + let drive = drive(target, 0.8); + let table = drive.warp_table().expect("in range"); + for (index, &code) in table.iter().enumerate() { + let phase = index as f64 / WARP_TABLE_LEN as f64; + let recovered = lobe_transmission(f64::from(code), &drive.inversion); + let target_u = drive.normalised_intensity(phase); + assert!( + (recovered - target_u).abs() < 5e-3, + "{target:?} phase {phase}: recovered {recovered} vs target {target_u}" + ); + } + } + } + + #[test] + fn measured_log_contrast_matches_the_requested_depth_for_log_sine() { + // The optical min/max of a log-sine table give back a = ln(max/min). + let drive = drive(OpticalTarget::LogSine, 1.2); + let table = drive.warp_table().expect("in range"); + let intensities: Vec = table + .iter() + .map(|&code| lobe_transmission(f64::from(code), &drive.inversion)) + .collect(); + let max = intensities.iter().cloned().fold(f64::MIN, f64::max); + let min = intensities.iter().cloned().fold(f64::MAX, f64::min); + let measured_a = (max / min).ln(); + assert!((measured_a - 1.2).abs() < 0.05, "measured a = {measured_a}"); + } + + #[test] + fn drive_derived_from_a_dac_band_recovers_that_optical_span() { + let inversion = inversion(); + let lo = 600.0; + let hi = 1_500.0; + let expected_lo = inversion.u_for_dac(lo); + let expected_hi = inversion.u_for_dac(hi); + + for target in [OpticalTarget::LogSine, OpticalTarget::LinearSine] { + let drive = OpticalDrive::from_dac_band(target, inversion, lo, hi); + let table = drive.warp_table().expect("manual band is valid"); + let recovered: Vec = table + .iter() + .map(|&code| inversion.u_for_dac(f64::from(code))) + .collect(); + let recovered_lo = recovered.iter().copied().fold(f64::MAX, f64::min); + let recovered_hi = recovered.iter().copied().fold(f64::MIN, f64::max); + assert!( + (recovered_lo - expected_lo).abs() < 5e-3, + "{target:?}: recovered lower intensity {recovered_lo} vs {expected_lo}" + ); + assert!( + (recovered_hi - expected_hi).abs() < 5e-3, + "{target:?}: recovered upper intensity {recovered_hi} vs {expected_hi}" + ); + } + } + + #[test] + fn deeper_depth_gives_more_optical_contrast() { + let contrast = |a: f64| { + let drive = drive(OpticalTarget::LinearSine, a); + let table = drive.warp_table().expect("in range"); + let intensities: Vec = table + .iter() + .map(|&code| lobe_transmission(f64::from(code), &drive.inversion)) + .collect(); + let max = intensities.iter().cloned().fold(f64::MIN, f64::max); + let min = intensities.iter().cloned().fold(f64::MAX, f64::min); + (max / min).ln() + }; + assert!(contrast(1.0) > contrast(0.5)); + } + + #[test] + fn rejects_invalid_depth_and_inversion() { + assert_eq!( + drive(OpticalTarget::LogSine, 0.0).warp_table(), + Err(WarpError::InvalidDepth) + ); + let mut bad = drive(OpticalTarget::LogSine, 1.0); + bad.inversion.v_pi_dac = 0.0; + assert_eq!(bad.warp_table(), Err(WarpError::InvalidInversion)); + } + + #[test] + fn refuses_an_inversion_that_overruns_the_lobe() { + // The reachable optical maximum sits at v_null + Vπ; pushing that past + // the top rail must be refused rather than silently clamped. + let drive = OpticalDrive { + target: OpticalTarget::LogSine, + depth_a: 1.0, + operating_point: peak_operating_point(OpticalTarget::LogSine, 1.0), + inversion: LobeInversion { + v_null_dac: 200.0, + v_pi_dac: 4_000.0, // peak light would land at code 4200 + }, + }; + assert!(matches!( + drive.warp_table(), + Err(WarpError::OutOfRange { .. }) + )); + } + + #[test] + fn refuses_an_operating_point_too_bright_for_the_depth() { + let drive = OpticalDrive { + target: OpticalTarget::LogSine, + depth_a: 1.0, + operating_point: 0.9, // 0.9 * exp(0.5) = 1.48 > 1 -> saturates + inversion: inversion(), + }; + assert!(matches!( + drive.warp_table(), + Err(WarpError::Saturates { .. }) + )); + } + + #[test] + fn fixed_operating_point_keeps_i_k_while_sweeping_a() { + // One response curve: fix u_k, vary a. The geometric-mean intensity at + // phase 0 (sin = 0) stays put; only the contrast grows with a. + let u_k = 0.3; + let drive = |a: f64| OpticalDrive { + target: OpticalTarget::LogSine, + depth_a: a, + operating_point: u_k, + inversion: inversion(), + }; + for a in [0.2, 0.6, 1.0] { + // At phase 0 the log-sine sits exactly at the operating point. + assert!((drive(a).normalised_intensity(0.0) - u_k).abs() < 1e-12); + let table = drive(a).warp_table().expect("in range"); + let intensities: Vec = table + .iter() + .map(|&code| lobe_transmission(f64::from(code), &inversion())) + .collect(); + let max = intensities.iter().cloned().fold(f64::MIN, f64::max); + let min = intensities.iter().cloned().fold(f64::MAX, f64::min); + assert!(((max / min).ln() - a).abs() < 0.05, "a={a}"); + } + } +} diff --git a/plugins/stage-a-photodiode/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml index a838ebb..3e0c8c0 100644 --- a/plugins/stage-a-photodiode/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -14,3 +14,4 @@ augur-plugin-api.workspace = true serde_json.workspace = true serialport.workspace = true stage-a-io = { path = "../../stage-a-io", default-features = false } +stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index af2e277..44d48d7 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -1,8 +1,8 @@ # Stage-A Photodiode Live readout of the photodiode on **board SMA5 → Teensy pin 18 / analog input A4**, from the -free-running ASCII stream the `stage-a-controller` firmware (0.3.0+) emits on its **second** USB -serial port (`PD code=… n=… t_ms=…` at 50 Hz). The port carries no commands, so this plugin is +free-running PDA1 `SamplesU16` stream the `stage-a-controller` firmware (0.4.0+) emits on its +**second** USB serial port (20 kSa/s default). The port carries no commands, so this plugin is read-only by construction; the command port belongs to `stage-a-modulation`. ## Modes @@ -16,11 +16,29 @@ read-only by construction; the command port belongs to `stage-a-modulation`. ## Views - a live rolling chart (window length settable, 1–120 s) of the value in the selected mode; -- a compact status table with the newest code/value and Connect/Disconnect actions. +- a compact status table with the newest code/value, moving average, integrity, + recording state, and connection state. ## Ports **Use `auto` (default recommendation):** it listens briefly on every attached usbmodem/ttyACM -device and connects to the one actually streaming `PD` lines — that is always the Teensy stream -port. Picking the command port manually by mistake is harmless: its binary frames parse to -nothing (no values appear). `mock` generates a synthetic slow sine for hardware-free testing. +device and connects to the one actually streaming CRC-clean PDA1 sample frames — that is always +the Teensy stream port. `mock` generates a synthetic sine for hardware-free testing. + +## Owner control service + +This plugin is the sole owner of the Teensy photodiode stream port. Workflow +plugins control named recordings through the versioned +`stage_a.photodiode.control.v1` service and consume bounded +`stage_a.photodiode_summary.v1` snapshots. They never open the serial port or +receive raw sample arrays through the control plane; finalized PDQ files remain +the replay and analysis source of truth. + +The snapshot's `stream.level` block carries the settled detector level over the +moving-average window in **raw** detector volts — the ADC map only, never the +RAW/EXCITATION display transform and never the optical geometry transform. It +also reports the window's peak-to-peak spread and the sample index it ends at, +so a consumer can prove a reading was taken *after* it changed something without +a shared clock. Unlike `optical_summary` it never refuses: it stays present +while the window clips (flagged), because the Pockels transfer sweep needs a +reading exactly where the reject-port detector is brightest. diff --git a/plugins/stage-a-photodiode/plugin.toml b/plugins/stage-a-photodiode/plugin.toml index 21b4bad..f773a84 100644 --- a/plugins/stage-a-photodiode/plugin.toml +++ b/plugins/stage-a-photodiode/plugin.toml @@ -1,3 +1,4 @@ +id = "stage-a.photodiode" name = "Stage-A Photodiode" version = "0.4.0" description = "Live photodiode readout (SMA5/pin 18/A4) from the Teensy stream port: raw values or excitation power I_exc = I_tot - I_pd." diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 1e0cbff..19313c7 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -21,10 +21,10 @@ //! or — for modulated signals — one full period of a user-given frequency, //! which makes the mean independent of the modulation phase. -use std::collections::VecDeque; -use std::fs::File; +use std::collections::{BTreeMap, VecDeque}; +use std::fs::{File, OpenOptions}; use std::io::{BufWriter, Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -34,12 +34,26 @@ use augur_plugin_api::PathDialogKind; use augur_plugin_api::{ export_plugin, EventStoreHandle, HostContext, HostDatasetDescriptor, HostDatasetKind, HostOutput, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, Plugin, - PluginFrame, Series1dLine, Series1dPoint, Series1dV1, SettingItem, SettingKind, SettingsSchema, - SettingsSection, StatusEntry, TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, - TableSchema, TableValueType, + PluginControlContext, PluginControlSnapshot, PluginFrame, PluginRuntimeRole, + PluginServiceOutcome, PluginServiceReply, PluginServiceRequest, Series1dLine, Series1dPoint, + Series1dV1, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, + TableColumn, TableColumnData, TableColumnValues, TableDatasetV1, TableSchema, TableValueType, }; use serde_json::{json, Value}; -use stage_a_io::{FrameParser, ParseEvent, PdqWriter, StreamIntegrity}; +use stage_a_io::{ + estimate_contrast, AdcCalibration, ContrastGeometry, FrameParser, ParseEvent, PdqWriter, + StreamIntegrity, +}; +use stage_a_plugin_contract::{ + ClientId, ConnectionStateV1, FreshnessV1, LeaseId, LeaseSnapshotV1, OwnerInstanceId, + PdqFinalizedReceiptV1, PdqReceiptV1, PdqStartSpecV1, PdqStartedReceiptV1, PdqTerminationV1, + PhotodiodeCalibrationV1, PhotodiodeCommandV1, PhotodiodeLevelV1, PhotodiodeOpticalSummaryV1, + PhotodiodeRequestV1, PhotodiodeResponseV1, PhotodiodeStreamV1, PhotodiodeSummaryV1, + RequestOutcomeV1, ResponseCommonV1, RunId, SampleRangeV1, SemanticRevision, ServiceErrorCodeV1, + ServiceErrorV1, Sha256V1, StreamIntegrityV1, SynchronizationV1, UnsyncedReasonV1, + CONTRACT_VERSION_V1, CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, PLUGIN_ID_STAGE_A_PHOTODIODE, + SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, +}; const SERIES_DATASET_ID: &str = "stage-a-photodiode.series"; const SPECTRUM_DATASET_ID: &str = "stage-a-photodiode.spectrum"; @@ -69,7 +83,22 @@ const SPECTRUM_MIN_SAMPLES: usize = 256; const SPECTRUM_MAX_SAMPLES: usize = 16_384; /// The firmware's default stream rate; the mock mirrors it. const MOCK_RATE_HZ: u32 = 20_000; +/// Trailing samples used for the live optical log-contrast `a`. Sized like the +/// spectrum window so a handful of modulation cycles are always covered. +const CONTRAST_WINDOW_SAMPLES: usize = 16_384; const MOCK_BLOCK_SAMPLES: usize = 256; +/// Cap on retained phase-0 markers (bounds the overlay + frequency window). +const MAX_MARKERS: usize = 4_096; +/// Mock phase-0 marker period in samples (20 kSa/s / 40 = 500 Hz modulation). +const MOCK_MARKER_PERIOD_SAMPLES: u64 = 40; +/// Codes within this margin of an ADC rail mark a level window as clipped; +/// mirrors the estimator's own clip margin. +const CLIP_MARGIN_CODES: u16 = 4; +const REQUEST_CACHE_LIMIT: usize = 256; +const MIN_LEASE_TTL_MS: u64 = 1_000; +const MAX_LEASE_TTL_MS: u64 = 60_000; +const SNAPSHOT_VALID_FOR_MS: u64 = 2_000; +static OWNER_SEQUENCE: AtomicU64 = AtomicU64::new(1); fn code_to_volts(code: f64) -> f64 { code * ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE @@ -138,6 +167,10 @@ struct SharedState { /// chart and moving average never rescan the raw window — at 500 kSa/s a /// full-window rescan per repaint would not be viable. cells: VecDeque, + /// Phase-0 marker sample indices (device clock) still inside the ring, from + /// `Marker` stream frames. Used for the opt-in trigger overlay and to derive + /// the modulation frequency. + markers: VecDeque, latest: Option, /// Cumulative firmware-side drop counter (latest header value). device_dropped: u32, @@ -148,6 +181,7 @@ struct SharedState { /// Monitor-cache length driving ring eviction (user setting). cache_seconds: f64, error: Option, + last_update_unix_ms: u64, } /// min/max/sum over exactly [`SUMMARY_CELL`] consecutive raw samples. @@ -183,6 +217,7 @@ impl Default for SharedState { ring_first_index: 0, samples: VecDeque::new(), cells: VecDeque::new(), + markers: VecDeque::new(), latest: None, device_dropped: 0, crc_failures: 0, @@ -190,6 +225,7 @@ impl Default for SharedState { segments: 0, cache_seconds: DEFAULT_CACHE_SECONDS, error: None, + last_update_unix_ms: 0, } } } @@ -215,12 +251,14 @@ impl SharedState { } self.samples.clear(); self.cells.clear(); + self.markers.clear(); self.ring_first_index = first_index; self.rate_hz = rate_hz; } self.samples.extend(codes.iter().copied()); self.latest = codes.last().copied(); self.device_dropped = device_dropped; + self.last_update_unix_ms = now_unix_ms(); // Summarize every newly completed cell. while (self.cells.len() + 1) * SUMMARY_CELL <= self.samples.len() { @@ -251,6 +289,47 @@ impl SharedState { self.cells.drain(..evict_cells); self.ring_first_index += evict as u64; } + // Drop markers that fell out of the retained ring window. + while self + .markers + .front() + .is_some_and(|&index| index < self.ring_first_index) + { + self.markers.pop_front(); + } + } + + /// Records a phase-0 marker (device sample index) if it sits inside the + /// current ring window. Bounded so a marker storm cannot grow unbounded. + fn push_marker(&mut self, sample_index: u64) { + if sample_index < self.ring_first_index { + return; + } + if self + .markers + .back() + .is_some_and(|&last| last == sample_index) + { + return; // ignore duplicate stamps + } + self.markers.push_back(sample_index); + while self.markers.len() > MAX_MARKERS { + self.markers.pop_front(); + } + self.last_update_unix_ms = now_unix_ms(); + } + + /// Mean marker spacing in samples, i.e. the modulation period on the device + /// clock — the trigger *defining* the frequency. `None` with < 2 markers. + fn marker_period_samples(&self) -> Option { + if self.markers.len() < 2 { + return None; + } + let first = *self.markers.front()?; + let last = *self.markers.back()?; + let spans = (self.markers.len() - 1) as f64; + let period = last.saturating_sub(first) as f64 / spans; + (period > 0.0).then_some(period) } /// min/max/sum over deque offsets `[start, end)`, combining whole @@ -306,6 +385,14 @@ impl SharedState { struct RecordingSink { writer: PdqWriter, pdq_path: PathBuf, + sidecar_path: PathBuf, + pdq_path_label: String, + sidecar_path_label: String, + run_id: RunId, + opened_at_unix_ms: u64, + stream_epoch: u64, + first_sample_index: Option, + metadata: BTreeMap, started_slug: String, samples_written: u64, write_error: Option, @@ -317,6 +404,19 @@ struct RecordingSink { start_segments: u64, } +impl RecordingSink { + fn started_receipt(&self) -> PdqStartedReceiptV1 { + PdqStartedReceiptV1 { + run_id: self.run_id.clone(), + pdq_path: self.pdq_path_label.clone(), + sidecar_path: self.sidecar_path_label.clone(), + opened_at_unix_ms: self.opened_at_unix_ms, + stream_epoch: self.stream_epoch, + first_sample_index: self.first_sample_index, + } + } +} + type SharedRecording = Arc>>; fn record_frame(recording: &SharedRecording, frame: &stage_a_io::Frame, samples: usize) { @@ -422,6 +522,15 @@ impl Reader { sequence = sequence.wrapping_add(1); if let Ok(mut state) = shared.lock() { state.ingest(next_index, MOCK_RATE_HZ, 0, &codes); + // Synthesize phase-0 markers on the device clock so the + // trigger overlay and frequency work without hardware. + let block_end = next_index + MOCK_BLOCK_SAMPLES as u64; + let mut marker = + next_index.next_multiple_of(MOCK_MARKER_PERIOD_SAMPLES); + while marker < block_end { + state.push_marker(marker); + marker += MOCK_MARKER_PERIOD_SAMPLES; + } } next_index += MOCK_BLOCK_SAMPLES as u64; produced = true; @@ -506,6 +615,13 @@ fn read_frames( while let Some(event) = parser.next_event() { match event { ParseEvent::Frame(frame) => { + if let Some(marker) = frame.marker() { + if let Ok(mut state) = shared.lock() { + state.push_marker(marker.sample_index); + } + changed = true; + continue; + } let Some(codes) = frame.samples() else { continue; // Control/summary frames are not expected here. }; @@ -540,6 +656,15 @@ fn read_frames( pub struct StageAPhotodiodePlugin { enabled: bool, + runtime_role: PluginRuntimeRole, + effects_allowed: bool, + owner_instance: OwnerInstanceId, + lease: Option, + request_cache: VecDeque<(PluginServiceRequest, PluginServiceReply)>, + requested_revision: Option, + acknowledged_revision: Option, + last_response: Option, + last_finalized_recording: Option, reader: Option, shared: Arc>, generation: Arc, @@ -556,13 +681,86 @@ pub struct StageAPhotodiodePlugin { avg_samples: usize, avg_sync_freq_hz: f64, time_axis: TimeAxis, + /// Overlay the phase-0 trigger markers on the chart (opt-in). + show_markers: bool, data_dir: String, + // -- momentary-button press forwarding (see PressLatch) -- + press_save_snapshot: PressLatch, + press_record_start: PressLatch, + press_record_stop: PressLatch, +} + +/// Forwards momentary button presses across the host's UI-mirror → live-worker +/// settings snapshot. A click arrives as `true` on the clicked instance; the +/// other instance only ever sees the snapshot value from `get_setting`, so the +/// press is transported as a monotonic counter and a counter advance counts as +/// one press edge. The first counter a fresh instance sees is adopted silently +/// so a reloaded worker does not replay old presses. Without this, an +/// unguarded button `set_setting` fires on every settings sync — the +/// "snapshot files kept appearing" bug. +#[derive(Debug, Default, Clone, Copy)] +struct PressLatch { + counter: u64, + seen: Option, +} + +impl PressLatch { + /// Interprets a settings write to this button; returns true on a press edge. + fn accept(&mut self, value: &Value) -> bool { + if value.as_bool() == Some(true) { + self.counter += 1; + self.seen = Some(self.counter); + return true; + } + let Some(incoming) = value.as_u64() else { + return false; + }; + match self.seen { + None => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + false + } + Some(seen) if incoming > seen => { + self.seen = Some(incoming); + self.counter = self.counter.max(incoming); + true + } + Some(_) => false, + } + } + + fn value(&self) -> Value { + json!(self.counter) + } +} + +#[derive(Clone)] +struct ControlLease { + lease_id: LeaseId, + holder: ClientId, + run_id: Option, + expires_at_unix_ms: u64, } impl Default for StageAPhotodiodePlugin { fn default() -> Self { Self { enabled: false, + runtime_role: PluginRuntimeRole::UiMirror, + effects_allowed: false, + owner_instance: OwnerInstanceId::new(format!( + "photodiode-{}-{}-{}", + std::process::id(), + now_unix_ms(), + OWNER_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )), + lease: None, + request_cache: VecDeque::new(), + requested_revision: None, + acknowledged_revision: None, + last_response: None, + last_finalized_recording: None, reader: None, shared: Arc::new(Mutex::new(SharedState::default())), generation: Arc::new(AtomicU64::new(1)), @@ -577,7 +775,11 @@ impl Default for StageAPhotodiodePlugin { avg_samples: 4, avg_sync_freq_hz: 0.0, time_axis: TimeAxis::BeforeNow, + show_markers: false, data_dir: String::new(), + press_save_snapshot: PressLatch::default(), + press_record_start: PressLatch::default(), + press_record_stop: PressLatch::default(), } } } @@ -591,6 +793,10 @@ impl StageAPhotodiodePlugin { if self.reader.is_some() { return; } + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + self.last_error = Some("connection deferred: hardware effects are not allowed".into()); + return; + } if let Ok(mut state) = self.shared.lock() { *state = SharedState::default(); } @@ -648,15 +854,189 @@ impl StageAPhotodiodePlugin { Ok(PathBuf::from(self.data_dir.trim())) } + /// Resolves a workflow-owned relative evidence path beneath the configured + /// data directory. Existing or newly created parent components must be + /// real directories, never symlinks. + fn resolve_control_path(&self, label: &str, extension: &str) -> Result { + let relative = Path::new(label); + if relative.as_os_str().is_empty() + || relative.is_absolute() + || relative + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err( + "workflow recording paths must be non-empty relative paths without '..'".into(), + ); + } + if relative.extension().and_then(|value| value.to_str()) != Some(extension) { + return Err(format!("workflow path must use the .{extension} extension")); + } + + let root = self.resolved_data_dir()?; + std::fs::create_dir_all(&root) + .map_err(|err| format!("creating {} failed: {err}", root.display()))?; + let root = root + .canonicalize() + .map_err(|err| format!("resolving data directory failed: {err}"))?; + let mut parent = root.clone(); + if let Some(relative_parent) = relative.parent() { + for component in relative_parent.components() { + let Component::Normal(name) = component else { + return Err("invalid workflow recording path".into()); + }; + parent.push(name); + match std::fs::symlink_metadata(&parent) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "workflow path crosses symlink {}", + parent.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!("{} is not a directory", parent.display())); + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(&parent).map_err(|err| { + format!("creating {} failed: {err}", parent.display()) + })?; + } + Err(err) => { + return Err(format!("checking {} failed: {err}", parent.display())); + } + } + let canonical = parent + .canonicalize() + .map_err(|err| format!("resolving {} failed: {err}", parent.display()))?; + if !canonical.starts_with(&root) { + return Err("workflow path escapes the configured data directory".into()); + } + } + } + let candidate = root.join(relative); + if let Ok(metadata) = std::fs::symlink_metadata(&candidate) { + if metadata.file_type().is_symlink() { + return Err(format!( + "workflow target is a symlink: {}", + candidate.display() + )); + } + } + Ok(candidate) + } + + fn begin_named_recording( + &mut self, + run_id: RunId, + specification: &PdqStartSpecV1, + ) -> Result { + if !self.connected() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "photodiode stream is not connected", + true, + )); + } + if specification.metadata.len() > 64 + || specification + .metadata + .iter() + .any(|(key, value)| key.len() > 128 || value.len() > 1_024) + { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "recording metadata exceeds owner bounds", + false, + )); + } + let (rate_hz, stream_epoch) = self + .shared + .lock() + .map(|state| (state.rate_hz, state.segments)) + .unwrap_or((0, 0)); + if specification + .expected_sample_rate_hz + .is_some_and(|expected| rate_hz != 0 && expected != rate_hz) + || specification + .expected_stream_epoch + .is_some_and(|expected| expected != stream_epoch) + { + return Err(service_error( + ServiceErrorCodeV1::Integrity, + "live photodiode stream does not match the requested epoch or sample rate", + true, + )); + } + let pdq_path = self + .resolve_control_path(&specification.pdq_path, "pdq") + .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; + let sidecar_path = self + .resolve_control_path(&specification.sidecar_path, "json") + .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; + if pdq_path == sidecar_path { + return Err(service_error( + ServiceErrorCodeV1::InvalidPath, + "PDQ and sidecar paths must differ", + false, + )); + } + self.open_recording( + run_id, + pdq_path, + sidecar_path, + specification.pdq_path.clone(), + specification.sidecar_path.clone(), + specification.metadata.clone(), + true, + ) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false)) + } + fn start_recording(&mut self) -> Result<(), String> { - if self.recording_active() { - return Ok(()); + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + return Err("recording is allowed only on the active live worker".into()); + } + if self.lease.is_some() { + return Err("manual recording is locked while a workflow lease is active".into()); } let dir = self.resolved_data_dir()?; let slug = timestamp_slug(); let pdq_path = dir.join(format!("pd_rec_{slug}.pdq")); - let writer = PdqWriter::create(&pdq_path) - .map_err(|err| format!("creating {} failed: {err}", pdq_path.display()))?; + let sidecar_path = pdq_path.with_extension("json"); + self.open_recording( + RunId::new(format!("manual-{slug}")), + pdq_path.clone(), + sidecar_path.clone(), + pdq_path.to_string_lossy().into_owned(), + sidecar_path.to_string_lossy().into_owned(), + BTreeMap::new(), + false, + )?; + self.last_save_note = Some(format!("recording → {}", pdq_path.display())); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn open_recording( + &mut self, + run_id: RunId, + pdq_path: PathBuf, + sidecar_path: PathBuf, + pdq_path_label: String, + sidecar_path_label: String, + metadata: BTreeMap, + exclusive: bool, + ) -> Result { + if self.recording_active() { + return Err("a photodiode recording is already active".into()); + } + let writer = if exclusive { + PdqWriter::create_new(&pdq_path) + } else { + PdqWriter::create(&pdq_path) + } + .map_err(|err| format!("creating {} failed: {err}", pdq_path.display()))?; let (crc, resync, dropped, segments) = match self.shared.lock() { Ok(state) => ( state.crc_failures, @@ -666,10 +1046,45 @@ impl StageAPhotodiodePlugin { ), Err(_) => (0, 0, 0, 0), }; + let (stream_epoch, first_sample_index) = self + .shared + .lock() + .map(|state| { + ( + state.segments, + (!state.samples.is_empty()) + .then_some(state.ring_first_index + state.samples.len() as u64), + ) + }) + .unwrap_or((0, None)); + let opened_at_unix_ms = now_unix_ms(); + let started_slug = timestamp_slug(); + if exclusive { + let started = json!({ + "kind": "recording_in_progress", + "run_id": run_id.as_str(), + "opened_at_unix_ms": opened_at_unix_ms, + "pdq_path": pdq_path_label, + "metadata": metadata, + }); + if let Err(err) = write_json_new(&sidecar_path, &started) { + drop(writer); + let _ = std::fs::remove_file(&pdq_path); + return Err(err); + } + } let sink = RecordingSink { writer, pdq_path: pdq_path.clone(), - started_slug: slug, + sidecar_path, + pdq_path_label, + sidecar_path_label, + run_id, + opened_at_unix_ms, + stream_epoch, + first_sample_index, + metadata, + started_slug, samples_written: 0, write_error: None, start_crc_failures: crc, @@ -677,16 +1092,25 @@ impl StageAPhotodiodePlugin { start_device_dropped: dropped, start_segments: segments, }; + let receipt = sink.started_receipt(); if let Ok(mut slot) = self.recording.lock() { *slot = Some(sink); } - self.last_save_note = Some(format!("recording → {}", pdq_path.display())); - Ok(()) + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(receipt) } fn stop_recording(&mut self) -> Result<(), String> { + self.finalize_recording(PdqTerminationV1::OperatorStopped) + .map(|_| ()) + } + + fn finalize_recording( + &mut self, + termination: PdqTerminationV1, + ) -> Result, String> { let Some(sink) = self.recording.lock().ok().and_then(|mut slot| slot.take()) else { - return Ok(()); + return Ok(None); }; let (rate_hz, crc, resync, dropped, segments) = match self.shared.lock() { Ok(state) => ( @@ -707,12 +1131,43 @@ impl StageAPhotodiodePlugin { let write_error = sink.write_error.clone(); let started = sink.started_slug.clone(); let samples = sink.samples_written; + let pdq_path = sink.pdq_path.clone(); + let sidecar_path = sink.sidecar_path.clone(); + let run_id = sink.run_id.clone(); + let opened_at_unix_ms = sink.opened_at_unix_ms; + let pdq_path_label = sink.pdq_path_label.clone(); + let sidecar_path_label = sink.sidecar_path_label.clone(); + let metadata = sink.metadata.clone(); let summary = sink .writer .finish(integrity) .map_err(|err| format!("finishing recording failed: {err}"))?; + let contract_integrity = contract_integrity(summary.integrity, summary.sample_segments); + let receipt = PdqFinalizedReceiptV1 { + run_id: run_id.clone(), + pdq_path: pdq_path_label, + sidecar_path: sidecar_path_label, + opened_at_unix_ms, + finalized_at_unix_ms: now_unix_ms(), + file_size_bytes: summary.bytes_written, + sha256: Sha256V1::parse(summary.file_sha256_hex()) + .map_err(|err| format!("invalid recording digest: {err}"))?, + frames_written: summary.frames_written, + sample_frames_written: summary.sample_frames_written, + sample_range: summary.sample_range.map(|range| SampleRangeV1 { + first_sample_index: range.first_sample_index, + end_sample_index_exclusive: range.end_sample_index_exclusive, + sample_count: range.sample_count, + }), + sample_rate_hz: summary.sample_rate_hz, + segment_count: summary.sample_segments, + integrity: contract_integrity, + termination, + valid: summary.valid && write_error.is_none(), + }; let sidecar = json!({ "kind": "recording", + "run_id": run_id, "started_utc": started, "stopped_utc": timestamp_slug(), "port": self.port_hint, @@ -722,6 +1177,9 @@ impl StageAPhotodiodePlugin { "pdq_frames": summary.frames_written, "pdq_bytes": summary.bytes_written, "pdq_crc32": summary.file_crc32, + "pdq_sha256": receipt.sha256.as_str(), + "metadata": metadata, + "termination": receipt.termination, "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), "reference_volts": self.reference_volts, @@ -734,21 +1192,473 @@ impl StageAPhotodiodePlugin { "valid": summary.valid && write_error.is_none(), "write_error": write_error, }); - let sidecar_path = sink.pdq_path.with_extension("json"); write_json(&sidecar_path, &sidecar)?; self.last_save_note = Some(format!( "saved recording {} ({} samples)", - sink.pdq_path.display(), + pdq_path.display(), samples )); + self.last_finalized_recording = Some(receipt.clone()); + self.generation.fetch_add(1, Ordering::Relaxed); + Ok(Some(receipt)) + } + + fn lease_snapshot(&self) -> Option { + self.lease.as_ref().map(|lease| LeaseSnapshotV1 { + lease_id: lease.lease_id.clone(), + holder: lease.holder.clone(), + expires_at_unix_ms: lease.expires_at_unix_ms, + run_id: lease.run_id.clone(), + }) + } + + fn require_lease(&self, request: &PhotodiodeRequestV1) -> Result<(), ServiceErrorV1> { + let lease = self.lease.as_ref().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::LeaseRequired, + "the photodiode owner requires an active automation lease", + false, + ) + })?; + if now_unix_ms() > lease.expires_at_unix_ms { + return Err(service_error( + ServiceErrorCodeV1::LeaseExpired, + "the photodiode automation lease expired", + false, + )); + } + if request.lease_id.as_ref() != Some(&lease.lease_id) + || request.requester != lease.holder + || request.run_id != lease.run_id + { + return Err(service_error( + ServiceErrorCodeV1::LeaseMismatch, + "request lease, holder, or run does not match the active lease", + false, + )); + } Ok(()) } + fn require_new_revision( + &self, + request: &PhotodiodeRequestV1, + ) -> Result { + let revision = request.requested_revision.ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "recording transitions require requested_revision", + false, + ) + })?; + if self + .requested_revision + .is_some_and(|current| revision <= current) + { + return Err(service_error( + ServiceErrorCodeV1::StaleRequest, + "requested_revision must be newer than the current photodiode state", + false, + )); + } + Ok(revision) + } + + fn immediate_response( + &mut self, + request: &PhotodiodeRequestV1, + receipt: Option, + ) -> PhotodiodeResponseV1 { + let response = PhotodiodeResponseV1 { + common: ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: request.request_id, + owner_instance: self.owner_instance.clone(), + run_id: request.run_id.clone(), + requested_revision: request.requested_revision, + acknowledged_revision: self.acknowledged_revision, + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(now_unix_ms()), + error: None, + }, + receipt, + }; + self.last_response = Some(response.clone()); + self.generation.fetch_add(1, Ordering::Relaxed); + response + } + + fn handle_photodiode_command( + &mut self, + request: &PhotodiodeRequestV1, + ) -> Result { + match &request.command { + PhotodiodeCommandV1::Connect => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "connection cannot be changed while leased", + false, + )); + } + self.connect_requested = true; + self.connect(); + if !self.connected() { + return Err(service_error( + ServiceErrorCodeV1::Transport, + self.last_error + .clone() + .unwrap_or_else(|| "photodiode connection failed".into()), + true, + )); + } + Ok(self.immediate_response(request, None)) + } + PhotodiodeCommandV1::Disconnect { + finalize_recording, + reason, + } => { + if self.lease.is_some() { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "use ReleaseLease while the owner is leased", + false, + )); + } + let receipt = if *finalize_recording { + self.finalize_recording(PdqTerminationV1::OperatorStopped) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .map(PdqReceiptV1::Finalized) + } else { + None + }; + self.connect_requested = false; + self.disconnect(); + self.last_error = Some(format!("disconnected by service: {reason}")); + Ok(self.immediate_response(request, receipt)) + } + PhotodiodeCommandV1::AcquireLease { ttl_ms } => { + let lease_id = request.lease_id.clone().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "AcquireLease requires lease_id", + false, + ) + })?; + if let Some(active) = &self.lease { + if active.lease_id != lease_id || active.holder != request.requester { + return Err(service_error( + ServiceErrorCodeV1::LeaseBusy, + "the photodiode owner is already leased", + true, + )); + } + } + self.lease = Some(ControlLease { + lease_id, + holder: request.requester.clone(), + run_id: request.run_id.clone(), + expires_at_unix_ms: lease_deadline(*ttl_ms), + }); + Ok(self.immediate_response(request, None)) + } + PhotodiodeCommandV1::RenewLease { ttl_ms } => { + self.require_lease(request)?; + if let Some(lease) = &mut self.lease { + lease.expires_at_unix_ms = lease_deadline(*ttl_ms); + } + Ok(self.immediate_response(request, None)) + } + PhotodiodeCommandV1::ReleaseLease { + finalize_recording, + reason, + } => { + self.require_lease(request)?; + let receipt = if *finalize_recording { + self.finalize_recording(PdqTerminationV1::OperatorStopped) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .map(PdqReceiptV1::Finalized) + } else if self.recording_active() { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "cannot release a lease with an active recording unless it is finalized", + false, + )); + } else { + None + }; + self.lease = None; + self.last_error = Some(format!("automation lease released: {reason}")); + Ok(self.immediate_response(request, receipt)) + } + PhotodiodeCommandV1::BeginRecording { specification } => { + self.require_lease(request)?; + let revision = self.require_new_revision(request)?; + let run_id = request.run_id.clone().ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "BeginRecording requires run_id", + false, + ) + })?; + let started = self.begin_named_recording(run_id, specification)?; + self.requested_revision = Some(revision); + self.acknowledged_revision = Some(revision); + Ok(self.immediate_response(request, Some(PdqReceiptV1::Started(started)))) + } + PhotodiodeCommandV1::FinalizeRecording { termination } => { + self.require_lease(request)?; + let revision = self.require_new_revision(request)?; + let finalized = self + .finalize_recording(*termination) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "no photodiode recording is active", + false, + ) + })?; + self.requested_revision = Some(revision); + self.acknowledged_revision = Some(revision); + Ok(self.immediate_response(request, Some(PdqReceiptV1::Finalized(finalized)))) + } + PhotodiodeCommandV1::AbortRecording { reason } => { + self.require_lease(request)?; + let revision = self.require_new_revision(request)?; + let finalized = self + .finalize_recording(PdqTerminationV1::Aborted) + .map_err(|message| service_error(ServiceErrorCodeV1::Io, message, false))? + .ok_or_else(|| { + service_error( + ServiceErrorCodeV1::InvalidCommand, + "no photodiode recording is active", + false, + ) + })?; + self.requested_revision = Some(revision); + self.acknowledged_revision = Some(revision); + self.last_error = Some(format!("recording aborted: {reason}")); + Ok(self.immediate_response(request, Some(PdqReceiptV1::Finalized(finalized)))) + } + } + } + + /// Live optical log-contrast `a` from the trailing ring window. The ADC + /// always measures the rejected diode `I_pd`, so the display mode selects + /// the geometry: RAW reports the raw detector contrast (`Direct`), + /// EXCITATION reports the excitation contrast (`RejectedComplement`) using + /// `reference_volts` as the total-power anchor `I_tot`. `None` when there is + /// no valid window or, in EXCITATION mode, no valid anchor. + fn optical_summary(&self, samples: &VecDeque) -> Option { + let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); + let window: Vec = samples.iter().skip(start).copied().collect(); + let calibration = AdcCalibration { + volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, + offset_volts: 0.0, + dark_volts: 0.0, + full_scale_code: ADC_MAX_CODE as u16, + }; + let geometry = match self.mode { + Mode::Raw => ContrastGeometry::Direct, + Mode::Excitation => ContrastGeometry::RejectedComplement { + total_power_volts: self.reference_volts, + }, + }; + let estimate = estimate_contrast(&window, &calibration, geometry).ok()?; + let run_id = self + .lease + .as_ref() + .and_then(|lease| lease.run_id.clone()) + .unwrap_or_else(|| RunId::from("live")); + Some(PhotodiodeOpticalSummaryV1 { + run_id, + calibration: PhotodiodeCalibrationV1 { + adc_calibration_id: "adc-default".into(), + dark_id: "dark-0".into(), + anchor_id: match self.mode { + Mode::Raw => "detector-direct".into(), + Mode::Excitation => "reference-volts".into(), + }, + dark_volts: calibration.dark_volts, + total_power_volts: self.reference_volts, + }, + measured_log_contrast: estimate.a, + log_contrast_stddev: None, + excitation_min_volts: estimate.v_min_volts, + excitation_max_volts: estimate.v_max_volts, + excitation_headroom_volts: estimate.v_min_volts, + low_clip_fraction: estimate.low_clip_fraction, + high_clip_fraction: estimate.high_clip_fraction, + measured_frequency_hz: None, + fundamental_phase_rad: None, + total_harmonic_distortion: None, + }) + } + + /// Locks the ring and returns the current optical log-contrast summary. + fn latest_optical(&self) -> Option { + let state = self.shared.lock().ok()?; + self.optical_summary(&state.samples) + } + + fn control_summary(&self) -> PhotodiodeSummaryV1 { + let (stream, connection, observed_at, optical_summary) = match self.shared.lock() { + Ok(state) => { + let sample_range = (!state.samples.is_empty()).then_some(SampleRangeV1 { + first_sample_index: state.ring_first_index, + end_sample_index_exclusive: state.ring_first_index + state.samples.len() as u64, + sample_count: state.samples.len() as u64, + }); + let optical_summary = self.optical_summary(&state.samples); + let level = self.current_level(&state); + let connection = if self.connected() { + ConnectionStateV1::Connected { + port_label: self.port_hint.clone(), + firmware_version: None, + } + } else if let Some(message) = + state.error.clone().or_else(|| self.last_error.clone()) + { + ConnectionStateV1::Faulted { message } + } else if self.connect_requested { + ConnectionStateV1::Connecting + } else { + ConnectionStateV1::Disconnected + }; + ( + PhotodiodeStreamV1 { + stream_epoch: state.segments, + sample_range, + sample_rate_hz: (state.rate_hz != 0).then_some(state.rate_hz), + latest_adc_code: state.latest, + integrity: StreamIntegrityV1 { + skipped_bytes: state.resync_bytes, + crc_failures: state.crc_failures, + sequence_gaps: state.segments, + dropped_samples: u64::from(state.device_dropped), + segment_restarts: state.segments, + truncated_bytes: 0, + }, + level, + }, + connection, + state.last_update_unix_ms, + optical_summary, + ) + } + Err(_) => ( + PhotodiodeStreamV1 { + stream_epoch: 0, + sample_range: None, + sample_rate_hz: None, + latest_adc_code: None, + integrity: StreamIntegrityV1::default(), + level: None, + }, + ConnectionStateV1::Faulted { + message: "photodiode state lock poisoned".into(), + }, + 0, + None, + ), + }; + let active_recording = self + .recording + .lock() + .ok() + .and_then(|slot| slot.as_ref().map(RecordingSink::started_receipt)); + let synchronization = match ( + self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + self.requested_revision, + self.acknowledged_revision, + ) { + (Some(run_id), Some(requested), Some(acknowledged)) if requested == acknowledged => { + SynchronizationV1::Synced { + run_id, + acknowledged_revision: acknowledged, + stream_epoch: Some(stream.stream_epoch), + } + } + (None, _, _) => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::NoLease, + detail: None, + }, + _ => SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::RequestedRevisionNotAcknowledged, + detail: None, + }, + }; + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: self.owner_instance.clone(), + service_revision: self.generation.load(Ordering::Relaxed), + connection, + lease: self.lease_snapshot(), + active_run_id: self.lease.as_ref().and_then(|lease| lease.run_id.clone()), + requested_revision: self.requested_revision, + acknowledged_revision: self.acknowledged_revision, + stream, + active_recording, + last_finalized_recording: self.last_finalized_recording.clone(), + optical_summary, + synchronization, + last_response: self.last_response.clone(), + freshness: FreshnessV1 { + observed_at_unix_ms: if observed_at == 0 { + now_unix_ms() + } else { + observed_at + }, + valid_for_ms: SNAPSHOT_VALID_FOR_MS, + }, + } + } + + fn expire_lease_if_needed(&mut self) { + if self + .lease + .as_ref() + .is_none_or(|lease| now_unix_ms() <= lease.expires_at_unix_ms) + { + return; + } + if let Err(error) = self.finalize_recording(PdqTerminationV1::LeaseExpired) { + self.last_error = Some(error); + } else { + self.last_error = Some("automation lease expired; recording finalized".into()); + } + self.lease = None; + self.generation.fetch_add(1, Ordering::Relaxed); + } + + fn apply_execution_context(&mut self, execution: &augur_plugin_api::ExecutionContext) { + let allowed = self.runtime_role == PluginRuntimeRole::LiveWorker + && execution.hardware_effects_allowed(); + self.effects_allowed = allowed; + if !allowed { + if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { + self.last_error = Some(error); + } + self.connect_requested = false; + self.disconnect(); + self.lease = None; + return; + } + self.expire_lease_if_needed(); + if self.connect_requested && self.reader.is_none() { + self.connect(); + } + } + /// Dumps the current monitor cache (ring) as CSV + JSON sidecar. Raw /// codes and raw volts only — mode/reference land in the sidecar so /// EXCITATION values stay derivable without baking display state into /// the data. fn save_cache_snapshot(&mut self) -> Result<(), String> { + if self.runtime_role != PluginRuntimeRole::LiveWorker || !self.effects_allowed { + return Err("saving is allowed only on the active live worker".into()); + } let dir = self.resolved_data_dir()?; let slug = timestamp_slug(); let csv_path = dir.join(format!("pd_cache_{slug}.csv")); @@ -844,6 +1754,40 @@ impl StageAPhotodiodePlugin { Some(state.range_summary(start, state.samples.len()).mean()) } + /// Settled detector level over the same window, published on the contract + /// in **raw** detector volts — never `display_volts`, so a consumer does + /// not have to know the display mode, and never the optical geometry + /// transform, which needs an anchor this reading must not depend on. + /// + /// Deliberately fail-open where [`Self::optical_summary`] is fail-closed: + /// a transfer-curve sweep needs a level exactly at the excitation null, + /// where the reject-port detector is brightest and may rail. Clipping is + /// reported rather than refused. + fn current_level(&self, state: &SharedState) -> Option { + if state.samples.is_empty() { + return None; + } + let window = self + .avg_window_samples(state.rate_hz) + .min(state.samples.len()); + let start = state.samples.len() - window; + let summary = state.range_summary(start, state.samples.len()); + if summary.count == 0 { + return None; + } + let full_scale = ADC_MAX_CODE as u16; + Some(PhotodiodeLevelV1 { + mean_volts: code_to_volts(summary.mean()), + // `code_to_volts` is a pure scale, so it maps a code difference to + // a voltage difference directly. + peak_to_peak_volts: code_to_volts(f64::from(summary.max - summary.min)), + sample_count: summary.count as u64, + end_sample_index: state.ring_first_index + state.samples.len() as u64, + clipped: summary.min <= CLIP_MARGIN_CODES + || summary.max >= full_scale.saturating_sub(CLIP_MARGIN_CODES), + }) + } + fn series_dataset(&self) -> Series1dV1 { let y_label = match self.mode { Mode::Raw => "photodiode [V]", @@ -952,6 +1896,43 @@ impl StageAPhotodiodePlugin { points: avg_points, }); } + // Opt-in phase-0 trigger overlay: one toggleable line drawing a vertical + // spike at each marker (up then back to a flat baseline between markers). + if self.show_markers && !state.markers.is_empty() { + let first_visible = state.ring_first_index + start as u64; + let y_range = lines + .iter() + .flat_map(|line| line.points.iter()) + .map(|point| point.y) + .fold(None::<(f64, f64)>, |acc, y| { + Some(acc.map_or((y, y), |(lo, hi)| (lo.min(y), hi.max(y)))) + }); + if let Some((y_lo, y_hi)) = y_range { + let x_for = |index: u64| -> f64 { + let device_t = index as f64 / rate; + match self.time_axis { + TimeAxis::BeforeNow => device_t - latest_x_index as f64 / rate, + TimeAxis::Segment => device_t, + } + }; + let mut points = Vec::with_capacity(state.markers.len() * 3); + for &index in &state.markers { + if index < first_visible || index > latest_x_index { + continue; + } + let x = x_for(index); + points.push(Series1dPoint { x, y: y_lo }); + points.push(Series1dPoint { x, y: y_hi }); + points.push(Series1dPoint { x, y: y_lo }); + } + if !points.is_empty() { + lines.push(Series1dLine { + name: "phase-0 trigger".into(), + points, + }); + } + } + } Series1dV1 { x_label: x_label.into(), y_label: y_label.into(), @@ -1180,6 +2161,85 @@ fn write_json(path: &Path, value: &Value) -> Result<(), String> { std::fs::write(path, bytes).map_err(|err| format!("writing {} failed: {err}", path.display())) } +fn write_json_new(path: &Path, value: &Value) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| format!("serializing sidecar failed: {err}"))?; + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|err| format!("creating {} failed: {err}", path.display()))?; + file.write_all(&bytes) + .and_then(|()| file.flush()) + .map_err(|err| format!("writing {} failed: {err}", path.display())) +} + +fn now_unix_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0) +} + +fn lease_deadline(ttl_ms: u64) -> u64 { + now_unix_ms().saturating_add(ttl_ms.clamp(MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS)) +} + +fn service_error( + code: ServiceErrorCodeV1, + message: impl Into, + retryable: bool, +) -> ServiceErrorV1 { + ServiceErrorV1 { + code, + message: message.into(), + retryable, + } +} + +fn contract_integrity(integrity: StreamIntegrity, segments: u64) -> StreamIntegrityV1 { + StreamIntegrityV1 { + skipped_bytes: integrity.skipped_bytes, + crc_failures: integrity.crc_failures, + sequence_gaps: integrity.sequence_gaps, + dropped_samples: integrity.dropped_samples, + segment_restarts: segments.saturating_sub(1), + truncated_bytes: 0, + } +} + +fn accepted_service_reply( + request: &PluginServiceRequest, + response: &PhotodiodeResponseV1, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Accepted { + payload: serde_json::to_value(response).unwrap_or(Value::Null), + }, + } +} + +fn rejected_service_reply( + request: &PluginServiceRequest, + code: impl Into, + message: impl Into, +) -> PluginServiceReply { + PluginServiceReply { + request_id: request.request_id, + source_plugin_id: request.source_plugin_id.clone(), + target_plugin_id: request.target_plugin_id.clone(), + service: request.service.clone(), + outcome: PluginServiceOutcome::Rejected { + code: code.into(), + message: message.into(), + }, + } +} + fn serial_ports() -> Vec { serialport::available_ports() .map(|ports| { @@ -1348,10 +2408,29 @@ impl Plugin for StageAPhotodiodePlugin { self.connect_requested = false; // Finalize an active recording so the .pdq/.json pair is complete // even when the plugin is disabled mid-run. - if let Err(err) = self.stop_recording() { + let termination = if self.lease.is_some() { + PdqTerminationV1::Aborted + } else { + PdqTerminationV1::OperatorStopped + }; + if let Err(err) = self.finalize_recording(termination) { self.last_error = Some(err); } self.disconnect(); + self.lease = None; + } + } + + fn set_runtime_role(&mut self, role: PluginRuntimeRole) { + self.runtime_role = role; + if role != PluginRuntimeRole::LiveWorker { + if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { + self.last_error = Some(error); + } + self.connect_requested = false; + self.disconnect(); + self.lease = None; + self.effects_allowed = false; } } @@ -1366,12 +2445,116 @@ impl Plugin for StageAPhotodiodePlugin { &mut self, _frame: &PluginFrame<'_>, _output: &mut HostOutput<'_>, - _context: &mut HostContext<'_>, + context: &mut HostContext<'_>, _event_store: &EventStoreHandle<'_>, ) { - // Reading is settings-driven (connect checkbox) and works without - // camera frames; the stream port carries no commands, so no replay - // teardown is needed either. + if context.execution().mode == augur_plugin_api::ExecutionMode::Replay { + if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { + self.last_error = Some(error); + } + self.connect_requested = false; + self.disconnect(); + self.lease = None; + } + } + + fn process_control(&mut self, context: &mut PluginControlContext<'_>) { + let execution = context.execution(); + self.apply_execution_context(&execution); + } + + fn handle_service_request( + &mut self, + request: &PluginServiceRequest, + execution: &augur_plugin_api::ExecutionContext, + ) -> PluginServiceReply { + if let Some((previous, reply)) = self.request_cache.iter().find(|(previous, _)| { + previous.source_plugin_id == request.source_plugin_id + && previous.request_id == request.request_id + }) { + return if previous == request { + reply.clone() + } else { + rejected_service_reply( + request, + "request_id_conflict", + "request ID was reused for a different photodiode payload", + ) + }; + } + + let reply = if request.target_plugin_id != PLUGIN_ID_STAGE_A_PHOTODIODE { + rejected_service_reply(request, "wrong_target", "wrong photodiode owner target") + } else if request.service != SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1 { + rejected_service_reply( + request, + "unsupported_service", + format!("unsupported photodiode service '{}'", request.service), + ) + } else if self.runtime_role != PluginRuntimeRole::LiveWorker + || !execution.hardware_effects_allowed() + { + rejected_service_reply( + request, + "effects_not_allowed", + "photodiode effects are allowed only on the active live worker", + ) + } else { + self.effects_allowed = true; + match serde_json::from_value::(request.payload.clone()) { + Err(error) => rejected_service_reply( + request, + "invalid_payload", + format!("invalid photodiode request: {error}"), + ), + Ok(payload) + if payload.contract_version != CONTRACT_VERSION_V1 + || payload.request_id.0 != request.request_id + || payload.requester.as_str() != request.source_plugin_id + || payload + .target_owner_instance + .as_ref() + .is_some_and(|owner| owner != &self.owner_instance) => + { + rejected_service_reply( + request, + "identity_mismatch", + "contract version, request, requester, or owner instance mismatch", + ) + } + Ok(payload) + if payload.issued_at_unix_ms != 0 + && (now_unix_ms().saturating_sub(payload.issued_at_unix_ms) > 120_000 + || payload.issued_at_unix_ms.saturating_sub(now_unix_ms()) + > 30_000) => + { + rejected_service_reply(request, "stale_request", "request timestamp is stale") + } + Ok(payload) => match self.handle_photodiode_command(&payload) { + Ok(response) => accepted_service_reply(request, &response), + Err(error) => rejected_service_reply( + request, + format!("{:?}", error.code).to_ascii_lowercase(), + error.message, + ), + }, + } + }; + self.request_cache + .push_back((request.clone(), reply.clone())); + while self.request_cache.len() > REQUEST_CACHE_LIMIT { + self.request_cache.pop_front(); + } + reply + } + + fn control_snapshots(&self) -> Vec { + vec![PluginControlSnapshot { + plugin_id: PLUGIN_ID_STAGE_A_PHOTODIODE.into(), + topic: CTX_STAGE_A_PHOTODIODE_SUMMARY_V1.into(), + revision: self.generation.load(Ordering::Relaxed).max(1), + payload: serde_json::to_value(self.control_summary()).unwrap_or(Value::Null), + }] } fn settings_schema(&self) -> SettingsSchema { @@ -1516,6 +2699,19 @@ impl Plugin for StageAPhotodiodePlugin { .unwrap_or(0), }, }, + SettingItem { + key: "show_markers".into(), + label: "Show phase-0 trigger markers".into(), + tooltip: Some( + "Overlay the firmware phase-0 markers (device-clock MARKER frames) \ + as a toggleable vertical curve. Also defines the modulation \ + frequency from the marker spacing." + .into(), + ), + kind: SettingKind::Bool { + default: self.show_markers, + }, + }, ], }, SettingsSection { @@ -1561,26 +2757,37 @@ impl Plugin for StageAPhotodiodePlugin { }, }, SettingItem { - key: "record".into(), - label: "Record to disk".into(), + key: "record_start".into(), + label: "Start recording".into(), tooltip: Some( - "Start/stop appending every incoming sample frame to \ - pd_rec_.pdq; stopping writes the JSON sidecar." + "Start appending every incoming sample frame to \ + pd_rec_.pdq. Disabled until a data directory \ + is selected." .into(), ), - kind: SettingKind::Bool { - default: self.recording_active(), + kind: SettingKind::Button { + enabled: !self.data_dir.trim().is_empty(), }, }, + SettingItem { + key: "record_stop".into(), + label: "Stop recording".into(), + tooltip: Some( + "Stop the disk recording and write the JSON sidecar.".into(), + ), + kind: SettingKind::Button { enabled: true }, + }, SettingItem { key: "save_snapshot".into(), label: "Save cache snapshot".into(), tooltip: Some( - "Write the current cache as pd_cache_.csv \ - (+ JSON sidecar)." + "Write the current cache once as pd_cache_.csv \ + (+ JSON sidecar). Disabled until a data directory is selected." .into(), ), - kind: SettingKind::Button, + kind: SettingKind::Button { + enabled: !self.data_dir.trim().is_empty(), + }, }, ], }, @@ -1610,6 +2817,7 @@ impl Plugin for StageAPhotodiodePlugin { "reference_volts" => Some(json!(self.reference_volts)), "window_s" => Some(json!(self.window_s)), "avg_samples" => Some(json!(self.avg_samples)), + "show_markers" => Some(json!(self.show_markers)), "avg_sync_freq_hz" => Some(json!(self.avg_sync_freq_hz)), "time_axis" => { let index = TimeAxis::VARIANTS @@ -1624,14 +2832,25 @@ impl Plugin for StageAPhotodiodePlugin { .lock() .map(|state| state.cache_seconds) .unwrap_or(DEFAULT_CACHE_SECONDS))), + // Kept for compatibility (tests, external tooling); not in the + // schema anymore, so it is never synced across instances. "record" => Some(json!(self.recording_active())), - // Momentary trigger: never reports as pressed. - "save_snapshot" => Some(json!(false)), + // Button presses are exported as monotonic counters so the host's + // settings snapshot transports them from the UI mirror to the + // live worker (see PressLatch). + "record_start" => Some(self.press_record_start.value()), + "record_stop" => Some(self.press_record_stop.value()), + "save_snapshot" => Some(self.press_save_snapshot.value()), _ => None, } } fn set_setting(&mut self, key: &str, value: Value) -> Result<(), String> { + if self.lease.is_some() { + return Err(format!( + "manual setting '{key}' is locked while the photodiode owner is leased" + )); + } match key { "port" => { self.port_hint = variant_path(&enum_choice(&value, &port_variants())?).to_owned(); @@ -1660,6 +2879,10 @@ impl Plugin for StageAPhotodiodePlugin { self.reference_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); Ok(()) } + "show_markers" => { + self.show_markers = value.as_bool().ok_or("show_markers must be a boolean")?; + Ok(()) + } "window_s" => { let seconds = value.as_f64().ok_or("window_s must be a number")?; self.window_s = seconds.clamp(0.01, 120.0); @@ -1700,9 +2923,9 @@ impl Plugin for StageAPhotodiodePlugin { Ok(()) } "record" => { + // Compatibility alias (not in the schema): direct boolean + // start/stop with the same edge-free semantics as before. let requested = value.as_bool().ok_or("record must be a boolean")?; - // Failures surface through status entries (like `connect`), - // so a missing data directory doesn't read as a broken UI. let result = if requested { self.start_recording() } else { @@ -1716,12 +2939,39 @@ impl Plugin for StageAPhotodiodePlugin { self.generation.fetch_add(1, Ordering::Relaxed); Ok(()) } + "record_start" => { + // Failures surface through status entries (like `connect`), + // so a missing data directory doesn't read as a broken UI. + if self.press_record_start.accept(&value) { + match self.start_recording() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } + "record_stop" => { + if self.press_record_stop.accept(&value) { + match self.stop_recording() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } "save_snapshot" => { - match self.save_cache_snapshot() { - Ok(()) => self.last_error = None, - Err(err) => self.last_error = Some(err), + // Edge-guarded: the host re-applies the full settings snapshot + // on every sync, and an unguarded arm wrote one cache file per + // sync of *any* plugin's settings. + if self.press_save_snapshot.accept(&value) { + match self.save_cache_snapshot() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); } - self.generation.fetch_add(1, Ordering::Relaxed); Ok(()) } _ => Err(format!("unknown setting: {key}")), @@ -1771,6 +3021,27 @@ impl Plugin for StageAPhotodiodePlugin { ))); } } + if let Some(optical) = self.latest_optical() { + let label = match self.mode { + Mode::Raw => "a_raw (detector)", + Mode::Excitation => "a (excitation)", + }; + entries.push(StatusEntry::Text(format!( + "{label} = {:.3} (I {:.4}..{:.4} V)", + optical.measured_log_contrast, + optical.excitation_min_volts, + optical.excitation_max_volts + ))); + } + if let Ok(state) = self.shared.lock() { + if let Some(period_samples) = state.marker_period_samples() { + let hz = f64::from(state.rate_hz.max(1)) / period_samples; + entries.push(StatusEntry::Text(format!( + "Trigger: {} markers, f = {hz:.3} Hz", + state.markers.len() + ))); + } + } if self.recording_active() { let (samples, path) = self .recording @@ -1877,8 +3148,50 @@ export_plugin!(StageAPhotodiodePlugin); #[cfg(test)] mod tests { use super::*; + use augur_plugin_api::{ExecutionContext, ExecutionMode}; use stage_a_io::{Frame, FrameHeader, FrameType}; + fn live_execution() -> ExecutionContext { + ExecutionContext { + mode: ExecutionMode::LiveCapture, + effects_allowed: true, + session_id: Some("test".into()), + } + } + + fn live_plugin() -> StageAPhotodiodePlugin { + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_runtime_role(PluginRuntimeRole::LiveWorker); + plugin.effects_allowed = true; + plugin + } + + fn service_request( + plugin: &StageAPhotodiodePlugin, + id: u64, + requester: &str, + command: PhotodiodeCommandV1, + revision: Option, + ) -> PluginServiceRequest { + let mut payload = PhotodiodeRequestV1::new( + stage_a_plugin_contract::RequestId(id), + ClientId::from(requester), + command, + ); + payload.target_owner_instance = Some(plugin.owner_instance.clone()); + payload.run_id = Some(RunId::from("run-a")); + payload.lease_id = Some(LeaseId::from("lease-a")); + payload.requested_revision = revision.map(SemanticRevision); + payload.issued_at_unix_ms = now_unix_ms(); + PluginServiceRequest { + request_id: id, + source_plugin_id: requester.into(), + target_plugin_id: PLUGIN_ID_STAGE_A_PHOTODIODE.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + payload: serde_json::to_value(payload).unwrap(), + } + } + fn sample_frame(sequence: u32, first_index: u64, rate_hz: u32, codes: &[u16]) -> Vec { let payload: Vec = codes.iter().flat_map(|c| c.to_le_bytes()).collect(); Frame::build( @@ -1941,6 +3254,31 @@ mod tests { assert_eq!(state.segments, 2); } + #[test] + fn phase0_markers_define_frequency_and_evict_with_the_ring() { + // Ring holds 1 s = 20_000 samples at 20 kSa/s. + let mut state = SharedState { + cache_seconds: 1.0, + ..SharedState::default() + }; + // 500 Hz modulation: markers every 40 samples. + ingest_bytes(&mut state, &sample_frame(0, 0, 20_000, &[100; 40])); + state.push_marker(0); + state.push_marker(40); + state.push_marker(80); + assert_eq!(state.markers.len(), 3); + let period = state.marker_period_samples().expect("period"); + assert!((period - 40.0).abs() < 1e-9); + let hz = f64::from(state.rate_hz) / period; + assert!((hz - 500.0).abs() < 1e-6, "hz={hz}"); + + // Duplicate stamps are ignored, and markers before the ring start too. + state.push_marker(80); + state.ring_first_index = 60; + state.push_marker(40); // now below the ring start + assert_eq!(state.markers.len(), 3); + } + #[test] fn ring_is_bounded_by_duration() { let mut state = SharedState::default(); @@ -2001,6 +3339,35 @@ mod tests { assert!((average - 250.0).abs() < 1e-9); } + #[test] + fn published_level_is_raw_volts_and_survives_clipping() { + let mut plugin = StageAPhotodiodePlugin::default(); // window = 4 samples + let mut state = SharedState::default(); + state.ingest(0, 20_000, 0, &[0, 0, 0, 0, 100, 200, 300, 400]); + + let level = plugin.current_level(&state).expect("has samples"); + assert!((level.mean_volts - code_to_volts(250.0)).abs() < 1e-9); + assert!((level.peak_to_peak_volts - code_to_volts(300.0)).abs() < 1e-9); + assert_eq!(level.sample_count, 4); + // The window is the newest 4 of 8 ingested samples. + assert_eq!(level.end_sample_index, 8); + assert!(!level.clipped); + + // EXCITATION display must not leak into the published level: it stays + // the raw detector reading whatever the operator is looking at. + plugin.set_setting("mode", json!(1)).expect("excitation"); + let raw_again = plugin.current_level(&state).expect("has samples"); + assert_eq!(raw_again.mean_volts, level.mean_volts); + + // At the rail the optical summary refuses; the level must not, because + // that is exactly where a transfer sweep needs a reading. + let mut railed = SharedState::default(); + railed.ingest(0, 20_000, 0, &[4_095; 8]); + let clipped = plugin.current_level(&railed).expect("still reports"); + assert!(clipped.clipped); + assert!(plugin.optical_summary(&railed.samples).is_none()); + } + #[test] fn series_dataset_decimates_with_envelope_and_average() { let mut plugin = StageAPhotodiodePlugin::default(); @@ -2065,7 +3432,7 @@ mod tests { fn mock_reader_fills_the_ring_and_series() { let mut plugin = StageAPhotodiodePlugin { port_hint: "mock".into(), - ..Default::default() + ..live_plugin() }; plugin.connect(); let deadline = Instant::now() + Duration::from_secs(2); @@ -2212,7 +3579,7 @@ mod tests { #[test] fn cache_snapshot_writes_csv_and_sidecar() { let dir = temp_dir("snapshot"); - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = live_plugin(); plugin .set_setting("data_dir", json!(dir.display().to_string())) .unwrap(); @@ -2247,9 +3614,59 @@ mod tests { std::fs::remove_dir_all(dir).unwrap(); } + #[test] + fn forwarded_snapshot_counter_saves_exactly_once() { + let dir = temp_dir("snapshot-forwarded"); + let mut plugin = live_plugin(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + { + let mut state = plugin.shared.lock().unwrap(); + state.ingest(10, 20_000, 0, &[100, 200, 300]); + } + let csv_count = |dir: &std::path::Path| { + std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|ext| ext == "csv")) + .count() + }; + // First forwarded counter is the baseline a fresh worker adopts. + plugin.set_setting("save_snapshot", json!(2)).unwrap(); + assert_eq!(csv_count(&dir), 0, "baseline must not save"); + // One press on the mirror advances the counter by one → one file. + plugin.set_setting("save_snapshot", json!(3)).unwrap(); + assert_eq!(csv_count(&dir), 1); + // The host re-applies the same snapshot on every settings sync of any + // plugin — this used to write one file per sync. + plugin.set_setting("save_snapshot", json!(3)).unwrap(); + plugin.set_setting("save_snapshot", json!(3)).unwrap(); + assert_eq!(csv_count(&dir), 1, "re-applied snapshots must not save"); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn record_buttons_start_and_stop_the_disk_recording() { + let dir = temp_dir("record-buttons"); + let mut plugin = live_plugin(); + plugin + .set_setting("data_dir", json!(dir.display().to_string())) + .unwrap(); + plugin.set_setting("record_start", json!(true)).unwrap(); + assert!(plugin.recording_active()); + // Idle stop is a no-op, an active stop finalizes. + plugin.set_setting("record_stop", json!(true)).unwrap(); + assert!(!plugin.recording_active()); + assert!(plugin.last_error.is_none(), "{:?}", plugin.last_error); + plugin.set_setting("record_stop", json!(true)).unwrap(); + assert!(plugin.last_error.is_none()); + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn snapshot_without_data_dir_reports_an_error() { - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = live_plugin(); plugin.set_setting("save_snapshot", json!(true)).unwrap(); assert!(plugin .last_error @@ -2260,7 +3677,7 @@ mod tests { #[test] fn recording_tees_frames_to_pdq_and_writes_a_sidecar() { let dir = temp_dir("recording"); - let mut plugin = StageAPhotodiodePlugin::default(); + let mut plugin = live_plugin(); plugin .set_setting("data_dir", json!(dir.display().to_string())) .unwrap(); @@ -2342,4 +3759,217 @@ mod tests { .expect("name accepted"); assert_eq!(plugin.mode, Mode::Raw); } + + #[test] + fn ui_mirror_never_opens_the_stream_or_writes_recordings() { + let dir = temp_dir("ui-mirror"); + let mut plugin = StageAPhotodiodePlugin { + port_hint: "mock".into(), + data_dir: dir.display().to_string(), + ..Default::default() + }; + plugin.set_setting("connect", json!(true)).unwrap(); + plugin.set_setting("record", json!(true)).unwrap(); + assert!(!plugin.connected()); + assert!(!plugin.recording_active()); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn service_is_idempotent_and_enforces_exclusive_leases_without_frames() { + let mut plugin = live_plugin(); + let acquire = service_request( + &plugin, + 1, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + let first = plugin.handle_service_request(&acquire, &live_execution()); + let expiry = plugin.lease.as_ref().unwrap().expires_at_unix_ms; + let duplicate = plugin.handle_service_request(&acquire, &live_execution()); + assert_eq!(first, duplicate); + assert_eq!(plugin.lease.as_ref().unwrap().expires_at_unix_ms, expiry); + + let conflict = service_request( + &plugin, + 2, + "workflow-b", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&conflict, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + assert!(plugin.set_setting("mode", json!("RAW")).is_err()); + } + + #[test] + fn named_recording_rejects_unsafe_paths_and_returns_final_receipt() { + let dir = temp_dir("named"); + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.data_dir = dir.display().to_string(); + plugin.connect(); + let acquire = service_request( + &plugin, + 10, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&acquire, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + )); + + let unsafe_begin = service_request( + &plugin, + 11, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "../escape.pdq".into(), + sidecar_path: "run/escape.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + }, + }, + Some(1), + ); + assert!(matches!( + plugin + .handle_service_request(&unsafe_begin, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + + let begin = service_request( + &plugin, + 12, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "A1/run-a_pd.pdq".into(), + sidecar_path: "A1/run-a_pd.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::from([("workflow".into(), "A1".into())]), + }, + }, + Some(1), + ); + let begin_reply = plugin.handle_service_request(&begin, &live_execution()); + assert!(matches!( + begin_reply.outcome, + PluginServiceOutcome::Accepted { .. } + )); + assert_eq!( + plugin.handle_service_request(&begin, &live_execution()), + begin_reply, + "duplicate begin must not open a second file" + ); + record_frame(&plugin.recording, &mock_sample_frame(9, 0, &[1, 2, 3]), 3); + + let finalize = service_request( + &plugin, + 13, + "workflow-a", + PhotodiodeCommandV1::FinalizeRecording { + termination: PdqTerminationV1::Completed, + }, + Some(2), + ); + let reply = plugin.handle_service_request(&finalize, &live_execution()); + let PluginServiceOutcome::Accepted { payload } = reply.outcome else { + panic!("finalize rejected"); + }; + let response: PhotodiodeResponseV1 = serde_json::from_value(payload).unwrap(); + let Some(PdqReceiptV1::Finalized(receipt)) = response.receipt else { + panic!("missing finalized receipt"); + }; + assert_eq!(receipt.sha256.as_str().len(), 64); + assert!(receipt.file_size_bytes > 0); + assert!(dir.join(&receipt.pdq_path).is_file()); + assert!(dir.join(&receipt.sidecar_path).is_file()); + + let collision = service_request( + &plugin, + 14, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: receipt.pdq_path.clone(), + sidecar_path: receipt.sidecar_path.clone(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + }, + }, + Some(3), + ); + assert!(matches!( + plugin + .handle_service_request(&collision, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + plugin.disconnect(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn effects_revocation_finalizes_and_disconnects_without_a_frame() { + let dir = temp_dir("revoked"); + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.data_dir = dir.display().to_string(); + plugin.connect(); + let acquire = service_request( + &plugin, + 20, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + let begin = service_request( + &plugin, + 21, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "revoked/run.pdq".into(), + sidecar_path: "revoked/run.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + }, + }, + Some(1), + ); + plugin.handle_service_request(&begin, &live_execution()); + assert!(plugin.recording_active()); + + plugin.apply_execution_context(&ExecutionContext::fail_closed()); + assert!(!plugin.connected()); + assert!(!plugin.recording_active()); + assert!(plugin.lease.is_none()); + assert_eq!( + plugin + .last_finalized_recording + .as_ref() + .unwrap() + .termination, + PdqTerminationV1::Aborted + ); + std::fs::remove_dir_all(dir).unwrap(); + } } diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs index 809786a..1c87f44 100644 --- a/stage-a-io/src/estimator.rs +++ b/stage-a-io/src/estimator.rs @@ -1,11 +1,22 @@ //! Calibrated optical log-contrast estimator. //! -//! `a = ln(I_max / I_min)` is defined by the *measured light*, never by the -//! commanded DAC excursion: the Pockels-cell V→T response is non-linear, so -//! the photodiode ADC trace is the only valid source of `a` +//! `a = ln(I_exc,max / I_exc,min)` is defined by the *excitation light*, never +//! by the commanded DAC excursion: the Pockels-cell V→T response is non-linear, +//! so the photodiode ADC trace is the only valid source of `a` //! (knowledge base: `methodology/camera-calibration.md`, "define `a` from //! the light, not the drive"). //! +//! The detector geometry matters. When the photodiode sits behind the PBS +//! reject port it measures the *rejected complement* `I_pd = I_tot - I_exc`, +//! so the peak detector ratio is **not** the excitation contrast. The caller +//! selects the geometry via [`ContrastGeometry`]: +//! - [`ContrastGeometry::Direct`] — the detector already sees the excitation +//! intensity (e.g. the plugin's EXCITATION display, `I_tot - I_pd`), so +//! `a = ln(v_max / v_min)`. +//! - [`ContrastGeometry::RejectedComplement`] — the detector sees the rejected +//! light (the plugin's RAW display), so +//! `a = ln((I_tot - v_min) / (I_tot - v_max))`. +//! //! The estimator therefore: //! - converts ADC codes to volts through a characterised affine calibration, //! - subtracts the dark level (the detector is DC-coupled; `a` needs true @@ -13,8 +24,8 @@ //! - takes robust percentile extrema rather than raw min/max so single-code //! noise spikes do not bias the contrast, //! - refuses to produce a value at all when the window clips (top/bottom of -//! the ADC range) or has no headroom above dark — a wrong `a` is worse -//! than no `a`. +//! the ADC range), has no headroom above dark, or the total-power anchor is +//! below the measured signal — a wrong `a` is worse than no `a`. use serde::{Deserialize, Serialize}; @@ -48,10 +59,25 @@ impl AdcCalibration { } } +/// Optical geometry of the detector relative to the excitation beam. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub enum ContrastGeometry { + /// The detector already measures the excitation intensity, so + /// `a = ln(v_max / v_min)`. + Direct, + /// The detector sits behind the PBS reject port and measures the rejected + /// complement `I_pd = I_tot - I_exc`. `total_power_volts` is the + /// dark-corrected total power `I_tot`; the excitation contrast is + /// `a = ln((I_tot - v_min) / (I_tot - v_max))`. + RejectedComplement { total_power_volts: f64 }, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ContrastEstimate { - /// Peak-to-peak log-contrast `a = ln(V_max / V_min)` (dark-corrected). + /// Peak-to-peak excitation log-contrast `a = ln(I_exc,max / I_exc,min)` + /// (dark-corrected, geometry-resolved). pub a: f64, + /// Excitation intensity extrema in volts after the geometry transform. pub v_min_volts: f64, pub v_max_volts: f64, /// Fraction of samples at or below code 0 + margin. @@ -61,7 +87,7 @@ pub struct ContrastEstimate { pub sample_count: usize, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum EstimateError { /// Fewer samples than the estimator can use robustly. TooFewSamples { count: usize, minimum: usize }, @@ -72,6 +98,13 @@ pub enum EstimateError { }, /// The dark-corrected minimum is not positive: no optical headroom. NoHeadroomAboveDark, + /// The rejected-complement total-power anchor `I_tot` is not above the + /// measured detector maximum, so the excitation minimum would be + /// non-positive: the anchor is wrong or the light is not the complement. + TotalPowerBelowSignal { + total_power_volts: f64, + detector_max_volts: f64, + }, } impl std::fmt::Display for EstimateError { @@ -90,6 +123,14 @@ impl std::fmt::Display for EstimateError { Self::NoHeadroomAboveDark => { f.write_str("dark-corrected minimum is not positive; a is undefined") } + Self::TotalPowerBelowSignal { + total_power_volts, + detector_max_volts, + } => write!( + f, + "total-power anchor {total_power_volts:.4} V is not above the detector \ + maximum {detector_max_volts:.4} V; a is undefined" + ), } } } @@ -105,12 +146,15 @@ pub const MAX_CLIP_FRACTION: f64 = 0.001; const LOW_PERCENTILE: f64 = 0.01; const HIGH_PERCENTILE: f64 = 0.99; -/// Estimates the optical log-contrast from one settled, phase-attributed +/// Estimates the excitation log-contrast from one settled, phase-attributed /// ADC window. The window must span at least a few full modulation cycles; -/// enforcing that is the caller's job (it knows the drive frequency). +/// enforcing that is the caller's job (it knows the drive frequency). The +/// `geometry` selects whether the codes are the excitation intensity directly +/// or the rejected complement measured behind the PBS reject port. pub fn estimate_contrast( codes: &[u16], calibration: &AdcCalibration, + geometry: ContrastGeometry, ) -> Result { if codes.len() < MIN_SAMPLES { return Err(EstimateError::TooFewSamples { @@ -139,16 +183,38 @@ pub fn estimate_contrast( let low_code = percentile(&sorted, LOW_PERCENTILE); let high_code = percentile(&sorted, HIGH_PERCENTILE); - let v_min = calibration.code_to_volts(low_code) - calibration.dark_volts; - let v_max = calibration.code_to_volts(high_code) - calibration.dark_volts; - if v_min <= 0.0 || v_max <= 0.0 { - return Err(EstimateError::NoHeadroomAboveDark); - } + // Dark-corrected detector volts at the robust extrema. + let detector_low = calibration.code_to_volts(low_code) - calibration.dark_volts; + let detector_high = calibration.code_to_volts(high_code) - calibration.dark_volts; + + // Resolve the excitation extrema from the detector geometry. + let (exc_min, exc_max) = match geometry { + ContrastGeometry::Direct => { + if detector_low <= 0.0 { + return Err(EstimateError::NoHeadroomAboveDark); + } + (detector_low, detector_high) + } + ContrastGeometry::RejectedComplement { total_power_volts } => { + // The most transmitted excitation coincides with the least rejected + // light (detector_low), and vice versa. + if total_power_volts <= detector_high { + return Err(EstimateError::TotalPowerBelowSignal { + total_power_volts, + detector_max_volts: detector_high, + }); + } + ( + total_power_volts - detector_high, + total_power_volts - detector_low, + ) + } + }; Ok(ContrastEstimate { - a: (v_max / v_min).ln(), - v_min_volts: v_min, - v_max_volts: v_max, + a: (exc_max / exc_min).ln(), + v_min_volts: exc_min, + v_max_volts: exc_max, low_clip_fraction, high_clip_fraction, sample_count: codes.len(), @@ -183,7 +249,8 @@ mod tests { }; // center 2048, amplitude 900 -> dark-corrected V ratio: let codes = sine_codes(2_048.0, 900.0, 4_096); - let estimate = estimate_contrast(&codes, &calibration).expect("clean window estimates"); + let estimate = estimate_contrast(&codes, &calibration, ContrastGeometry::Direct) + .expect("clean window estimates"); let expected = ((2_048.0_f64 + 900.0 - 40.0) / (2_048.0 - 900.0 - 40.0)).ln(); assert!( @@ -194,11 +261,60 @@ mod tests { assert!(estimate.low_clip_fraction == 0.0 && estimate.high_clip_fraction == 0.0); } + #[test] + fn direct_and_rejected_complement_recover_the_same_excitation_contrast() { + // Excitation is a clean sine between exc_min and exc_max; the reject + // port sees the complement I_tot - I_exc. Both geometries must recover + // the same excitation log-contrast a = ln(exc_max / exc_min). + let calibration = AdcCalibration::default(); + let volts_per_code = calibration.volts_per_code; + let total_power_volts = 3_600.0 * volts_per_code; + let exc_center = 1_600.0; + let exc_amplitude = 900.0; + + let excitation_codes = sine_codes(exc_center, exc_amplitude, 4_096); + let rejected_codes: Vec = excitation_codes.iter().map(|&code| 3_600 - code).collect(); + + let direct = estimate_contrast(&excitation_codes, &calibration, ContrastGeometry::Direct) + .expect("direct excitation window"); + let rejected = estimate_contrast( + &rejected_codes, + &calibration, + ContrastGeometry::RejectedComplement { total_power_volts }, + ) + .expect("rejected complement window"); + + let expected = ((exc_center + exc_amplitude) / (exc_center - exc_amplitude)).ln(); + assert!((direct.a - expected).abs() < 0.01, "direct a={}", direct.a); + assert!( + (rejected.a - direct.a).abs() < 0.01, + "rejected a={} direct a={}", + rejected.a, + direct.a + ); + } + + #[test] + fn rejected_complement_rejects_a_total_power_anchor_below_the_signal() { + let calibration = AdcCalibration::default(); + let codes = sine_codes(2_048.0, 900.0, 2_048); + // Anchor far below the detector maximum (~2948 codes). + let err = estimate_contrast( + &codes, + &calibration, + ContrastGeometry::RejectedComplement { + total_power_volts: 1_000.0 * calibration.volts_per_code, + }, + ) + .expect_err("anchor below signal must be rejected"); + assert!(matches!(err, EstimateError::TotalPowerBelowSignal { .. })); + } + #[test] fn rejects_clipped_windows() { // Amplitude pushes past full scale -> clipping at the top rail. let codes = sine_codes(3_500.0, 900.0, 2_048); - let err = estimate_contrast(&codes, &AdcCalibration::default()) + let err = estimate_contrast(&codes, &AdcCalibration::default(), ContrastGeometry::Direct) .expect_err("clipped window must be rejected"); assert!(matches!(err, EstimateError::Clipped { .. })); } @@ -211,15 +327,19 @@ mod tests { }; // Minimum (2048-900=1148) sits below the dark level (1300). let codes = sine_codes(2_048.0, 900.0, 2_048); - let err = estimate_contrast(&codes, &calibration) + let err = estimate_contrast(&codes, &calibration, ContrastGeometry::Direct) .expect_err("no headroom above dark must be rejected"); assert_eq!(err, EstimateError::NoHeadroomAboveDark); } #[test] fn rejects_short_windows() { - let err = estimate_contrast(&[100; 10], &AdcCalibration::default()) - .expect_err("short window rejected"); + let err = estimate_contrast( + &[100; 10], + &AdcCalibration::default(), + ContrastGeometry::Direct, + ) + .expect_err("short window rejected"); assert!(matches!(err, EstimateError::TooFewSamples { .. })); } @@ -230,9 +350,12 @@ mod tests { let clean = estimate_contrast( &sine_codes(2_048.0, 500.0, 4_096), &AdcCalibration::default(), + ContrastGeometry::Direct, ) .expect("clean"); - let spiked = estimate_contrast(&codes, &AdcCalibration::default()).expect("spiked"); + let spiked = + estimate_contrast(&codes, &AdcCalibration::default(), ContrastGeometry::Direct) + .expect("spiked"); assert!((clean.a - spiked.a).abs() < 0.005); } } diff --git a/stage-a-io/src/lib.rs b/stage-a-io/src/lib.rs index 2a9eabd..bf726e7 100644 --- a/stage-a-io/src/lib.rs +++ b/stage-a-io/src/lib.rs @@ -12,7 +12,8 @@ //! ADC overruns — any of which invalidates a measurement point), //! - a bounded background I/O worker so plugin `process_frame()` never //! blocks on serial, -//! - the `.pdq` raw-frame writer and the JSON run sidecar, +//! - streaming `.pdq` write/replay with CRC32, SHA-256, byte/frame counts, +//! contiguous sample-range receipts, plus the JSON run sidecar, //! - the calibrated optical log-contrast estimator (`a` is measured light, //! never the commanded DAC excursion), //! - a mock controller for tests and hardware-free development. @@ -26,20 +27,29 @@ pub mod estimator; pub mod mock; pub mod pdq; pub mod protocol; +mod sha256; pub mod sidecar; pub mod transport; pub mod wire; pub use client::{ClientError, DeviceEvent, StageAClient, StreamIntegrity}; -pub use estimator::{estimate_contrast, AdcCalibration, ContrastEstimate, EstimateError}; +pub use estimator::{ + estimate_contrast, AdcCalibration, ContrastEstimate, ContrastGeometry, EstimateError, +}; pub use mock::{MockController, MockState, MockWave}; -pub use pdq::{PdqSummary, PdqWriter}; +pub use pdq::{ + inspect_pdq, PdqReadEvent, PdqReadSummary, PdqReader, PdqSampleRange, PdqSummary, PdqWriter, +}; pub use protocol::{Command, ControlMessage, ProtocolError}; +pub use sha256::Sha256Digest; pub use sidecar::{DetectorLoad, IntegrityRecord, RunSidecar, TriggerSource}; #[cfg(feature = "hardware")] pub use transport::SerialTransport; pub use transport::{MockLink, MockTransport, Transport}; -pub use wire::{Frame, FrameHeader, FrameParser, FrameType, ParseEvent, SummaryPayload}; +pub use wire::{ + Frame, FrameHeader, FrameParser, FrameType, MarkerPayload, ParseEvent, SummaryPayload, + MARKER_SOURCE_PHASE0, +}; pub use worker::{IoWorker, WorkerOutput, WorkerRequest}; pub mod worker; diff --git a/stage-a-io/src/mock.rs b/stage-a-io/src/mock.rs index 203733c..e3aabb0 100644 --- a/stage-a-io/src/mock.rs +++ b/stage-a-io/src/mock.rs @@ -70,6 +70,17 @@ impl MockWave { } } +/// Optical warp parameters accepted on `MOD wave=WARP` (firmware rebuilds the +/// DAC table from these; the mock only validates them). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct WarpParams { + target: Option, + a_milli: u32, + u_k_milli: u32, + v_null: u32, + v_pi: u32, +} + #[derive(Debug, Clone, PartialEq)] struct MockConfig { mode: String, @@ -435,6 +446,7 @@ impl MockController { let mut min_level = 0_u32; let mut freq_mhz = 0_u32; let mut saw_freq = false; + let mut warp: WarpParams = WarpParams::default(); for (key, value) in fields { match key.as_str() { "wave" => { @@ -443,6 +455,7 @@ impl MockController { "CONST" => "CONST", "SINE" => "SINE", "SQUARE" => "SQUARE", + "WARP" => "WARP", _ => return err("RANGE", "invalid_wave"), }); } @@ -464,12 +477,64 @@ impl MockController { } _ => return err("RANGE", "invalid_freq_mhz"), }, + // Optical warp parameters (wave=WARP): the firmware rebuilds the + // 256-entry DAC table from these; the mock only validates them. + "target" => match value.as_str() { + "LOG_SINE" | "LINEAR_SINE" => warp.target = Some(value.clone()), + _ => return err("RANGE", "invalid_target"), + }, + "a_milli" => match value.parse::() { + Ok(parsed) if parsed > 0 => warp.a_milli = parsed, + _ => return err("RANGE", "invalid_a"), + }, + "u_k_milli" => match value.parse::() { + Ok(parsed) if (1..=1_000).contains(&parsed) => warp.u_k_milli = parsed, + _ => return err("RANGE", "invalid_u_k"), + }, + "v_null" => match value.parse::() { + Ok(parsed) if parsed <= 4_095 => warp.v_null = parsed, + _ => return err("RANGE", "invalid_v_null"), + }, + "v_pi" => match value.parse::() { + Ok(parsed) if (1..=4_095).contains(&parsed) => warp.v_pi = parsed, + _ => return err("RANGE", "invalid_v_pi"), + }, _ => return err("SYNTAX", "unknown_mod_field"), } } let Some(wave) = wave else { return err("SYNTAX", "wave_required"); }; + if wave == "WARP" { + if !saw_freq { + return err("SYNTAX", "freq_mhz_required"); + } + if warp.target.is_none() { + return err("SYNTAX", "target_required"); + } + if warp.u_k_milli == 0 { + return err("SYNTAX", "u_k_required"); + } + if warp.v_null + warp.v_pi > 4_095 { + return err("RANGE", "warp_exceeds_range"); + } + if !(MOCK_MOD_MIN_FREQ_MHZ..=MOCK_MOD_MAX_FREQ_MHZ).contains(&freq_mhz) { + return err("RANGE", "mod_rejected"); + } + self.mod_wave = "WARP"; + self.mod_min = warp.v_null; + self.mod_level = warp.v_null + warp.v_pi; + self.mod_code = warp.v_null; + self.mod_freq_mhz = freq_mhz; + return format!( + "+{sequence} OK mod_wave=WARP mod_level={} mod_min={} mod_freq_mhz={} code={} target={}", + self.mod_level, + self.mod_min, + self.mod_freq_mhz, + self.mod_code, + warp.target.unwrap_or_default() + ); + } let periodic = wave == "SINE" || wave == "SQUARE"; if wave != "OFF" && !saw_level { return err("SYNTAX", "level_required"); @@ -777,6 +842,41 @@ mod tests { .contains("mod_wave=OFF mod_level=0 mod_min=0 mod_freq_mhz=0 code=0")); } + #[test] + fn mod_warp_validates_optical_parameters_and_reports_the_lobe_range() { + let link = MockLink::new(); + let mut host = link.host_end(); + let mut controller = MockController::new(link.device_end()); + + request(&mut controller, "@1 MOD wave=WARP freq_mhz=10000"); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=target_required")); + + // Operating point is required. + request( + &mut controller, + "@2 MOD wave=WARP freq_mhz=10000 target=LOG_SINE a_milli=800 v_null=200 v_pi=1600", + ); + assert!(last_control_text(&mut host).contains("code=SYNTAX detail=u_k_required")); + + // V_null + Vπ overruns the DAC top rail. + request( + &mut controller, + "@3 MOD wave=WARP freq_mhz=10000 target=LOG_SINE a_milli=800 u_k_milli=500 v_null=200 v_pi=4000", + ); + assert!(last_control_text(&mut host).contains("code=RANGE detail=warp_exceeds_range")); + + // A valid log-sine warp holds and reports the reachable code range. + request( + &mut controller, + "@4 MOD wave=WARP freq_mhz=10000 target=LOG_SINE a_milli=800 u_k_milli=500 v_null=200 v_pi=1600", + ); + let ok = last_control_text(&mut host); + assert!( + ok.contains("mod_wave=WARP mod_level=1800 mod_min=200 mod_freq_mhz=10000 code=200 target=LOG_SINE"), + "{ok}" + ); + } + #[test] fn waveform_extension_validates_drive_bounds() { let link = MockLink::new(); @@ -846,6 +946,7 @@ mod tests { dark_volts: 40.0 * 3.3 / 4_095.0, ..Default::default() }, + crate::estimator::ContrastGeometry::Direct, ) .expect("clean synthetic window"); estimate.a diff --git a/stage-a-io/src/pdq.rs b/stage-a-io/src/pdq.rs index 1eb4d1c..164ebf9 100644 --- a/stage-a-io/src/pdq.rs +++ b/stage-a-io/src/pdq.rs @@ -1,71 +1,387 @@ -//! `.pdq` writer: preserves every valid PDA1 frame verbatim on disk and -//! tracks run validity. +//! Streaming `.pdq` persistence, replay, and evidence receipts. //! -//! Raw ADC waveforms belong in the PDQ file, never in `HostContext` JSON or -//! per-frame plugin output. A CRC error, frame-sequence gap, or nonzero -//! dropped-sample counter invalidates the run — the file is still written -//! (evidence), but the sidecar must record `valid = false`. +//! Writers preserve every clean PDA1 frame verbatim and finalize the file +//! with CRC32, SHA-256, byte/frame counts, and a contiguous device sample +//! range when one exists. Readers incrementally recover the same frames, +//! report corruption/truncated tails, and produce an independently computed +//! summary suitable for replay verification. -use std::fs::File; -use std::io::{BufWriter, Write}; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use crate::client::StreamIntegrity; -use crate::wire::{Crc32, Frame}; +use crate::sha256::{Sha256, Sha256Digest}; +use crate::wire::{Crc32, Frame, FrameParser, FrameType, ParseEvent}; -pub struct PdqWriter { - path: PathBuf, - file: BufWriter, - frames_written: u64, - bytes_written: u64, - running_crc: Crc32, +const READ_BUFFER_BYTES: usize = 64 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PdqSampleRange { + pub first_sample_index: u64, + pub end_sample_index_exclusive: u64, + pub sample_count: u64, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct PdqSummary { pub path: PathBuf, pub frames_written: u64, + pub sample_frames_written: u64, + pub samples_written: u64, pub bytes_written: u64, - /// CRC32 over the whole file contents, recorded in the sidecar. + /// CRC32 over the complete file contents, retained for compatibility + /// with existing sidecars and quick local checks. pub file_crc32: u32, + /// SHA-256 over the complete file contents for immutable run receipts. + pub file_sha256: Sha256Digest, + /// Present only when every sample frame belongs to one contiguous, + /// constant-rate device-index segment. + pub sample_range: Option, + pub sample_rate_hz: Option, + pub sample_segments: u64, pub integrity: StreamIntegrity, pub valid: bool, } +impl PdqSummary { + pub fn file_sha256_hex(&self) -> String { + self.file_sha256.to_hex() + } +} + +pub struct PdqWriter { + path: PathBuf, + file: BufWriter, + frames_written: u64, + bytes_written: u64, + running_crc: Crc32, + running_sha256: Sha256, + tracker: FrameTracker, +} + impl PdqWriter { - pub fn create(path: impl AsRef) -> std::io::Result { - let path = path.as_ref().to_owned(); + pub fn create(path: impl AsRef) -> io::Result { + Self::create_with(path.as_ref(), false) + } + + /// Creates a new evidence file without replacing an existing run. + pub fn create_new(path: impl AsRef) -> io::Result { + Self::create_with(path.as_ref(), true) + } + + fn create_with(path: &Path, exclusive: bool) -> io::Result { + let path = path.to_owned(); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } + let file = OpenOptions::new() + .write(true) + .create(true) + .create_new(exclusive) + .truncate(!exclusive) + .open(&path)?; Ok(Self { - file: BufWriter::new(File::create(&path)?), + file: BufWriter::new(file), path, frames_written: 0, bytes_written: 0, running_crc: Crc32::default(), + running_sha256: Sha256::default(), + tracker: FrameTracker::default(), }) } - pub fn write_frame(&mut self, frame: &Frame) -> std::io::Result<()> { + pub fn write_frame(&mut self, frame: &Frame) -> io::Result<()> { let bytes = frame.to_bytes(); self.file.write_all(&bytes)?; self.frames_written += 1; self.bytes_written += bytes.len() as u64; self.running_crc.update(&bytes); + self.running_sha256.update(&bytes); + self.tracker.observe(frame); Ok(()) } - /// Flushes and closes the file, returning the summary for the sidecar. - pub fn finish(mut self, integrity: StreamIntegrity) -> std::io::Result { + /// Flushes and closes the file, returning everything needed for a named + /// finalized receipt. Integrity observed by the live transport is merged + /// fail-closed with discontinuities inferable from the written frames. + pub fn finish(mut self, mut integrity: StreamIntegrity) -> io::Result { self.file.flush()?; + integrity.sequence_gaps = integrity + .sequence_gaps + .max(self.tracker.frame_sequence_gaps); + integrity.dropped_samples = integrity + .dropped_samples + .max(self.tracker.dropped_samples_delta()); + let sample_range = self.tracker.contiguous_sample_range(); + let valid = integrity.is_clean() + && self.tracker.malformed_sample_frames == 0 + && self.tracker.sample_segments <= 1; Ok(PdqSummary { file_crc32: self.running_crc.finalize(), + file_sha256: self.running_sha256.finalize(), path: self.path, frames_written: self.frames_written, + sample_frames_written: self.tracker.sample_frames, + samples_written: self.tracker.samples, bytes_written: self.bytes_written, - valid: integrity.is_clean(), + sample_range, + sample_rate_hz: self.tracker.uniform_sample_rate(), + sample_segments: self.tracker.sample_segments, integrity, + valid, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PdqReadEvent { + Frame(Frame), + Corruption { + skipped_bytes: usize, + crc_failures: usize, + }, + TruncatedTail { + bytes: usize, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PdqReadSummary { + pub frames_read: u64, + pub sample_frames_read: u64, + pub samples_read: u64, + pub bytes_read: u64, + pub file_crc32: u32, + pub file_sha256: Sha256Digest, + pub sample_range: Option, + pub sample_rate_hz: Option, + pub sample_segments: u64, + pub malformed_sample_frames: u64, + pub truncated_bytes: u64, + pub integrity: StreamIntegrity, + pub valid: bool, +} + +impl PdqReadSummary { + pub fn file_sha256_hex(&self) -> String { + self.file_sha256.to_hex() + } +} + +/// Incremental PDA1 file reader. `next_event` preserves corruption notices +/// instead of silently skipping them, allowing replay to continue while the +/// final summary remains invalid. +pub struct PdqReader { + reader: R, + parser: FrameParser, + buffer: Vec, + eof: bool, + tail_reported: bool, + frames_read: u64, + bytes_read: u64, + running_crc: Crc32, + running_sha256: Sha256, + tracker: FrameTracker, + integrity: StreamIntegrity, + truncated_bytes: u64, +} + +impl PdqReader { + pub fn open(path: impl AsRef) -> io::Result { + File::open(path).map(Self::new) + } +} + +impl PdqReader { + pub fn new(reader: R) -> Self { + Self { + reader, + parser: FrameParser::default(), + buffer: vec![0; READ_BUFFER_BYTES], + eof: false, + tail_reported: false, + frames_read: 0, + bytes_read: 0, + running_crc: Crc32::default(), + running_sha256: Sha256::default(), + tracker: FrameTracker::default(), + integrity: StreamIntegrity::default(), + truncated_bytes: 0, + } + } + + pub fn next_event(&mut self) -> io::Result> { + loop { + if let Some(event) = self.parser.next_event() { + return Ok(Some(match event { + ParseEvent::Frame(frame) => { + self.frames_read += 1; + self.tracker.observe(&frame); + PdqReadEvent::Frame(frame) + } + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + self.integrity.skipped_bytes += skipped_bytes as u64; + self.integrity.crc_failures += crc_failures as u64; + PdqReadEvent::Corruption { + skipped_bytes, + crc_failures, + } + } + })); + } + + if self.eof { + if !self.tail_reported && self.parser.buffered_len() > 0 { + self.tail_reported = true; + let bytes = self.parser.discard_buffered(); + self.truncated_bytes += bytes as u64; + self.integrity.skipped_bytes += bytes as u64; + return Ok(Some(PdqReadEvent::TruncatedTail { bytes })); + } + return Ok(None); + } + + let read = match self.reader.read(&mut self.buffer) { + Ok(read) => read, + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Err(err), + }; + if read == 0 { + self.eof = true; + continue; + } + let bytes = &self.buffer[..read]; + self.bytes_read += read as u64; + self.running_crc.update(bytes); + self.running_sha256.update(bytes); + self.parser.extend(bytes); + } + } + + /// Drains the remaining file and returns an independently verified + /// summary. This can follow any number of prior `next_event` calls. + pub fn finish(mut self) -> io::Result { + while self.next_event()?.is_some() {} + self.integrity.sequence_gaps = self.tracker.frame_sequence_gaps; + self.integrity.dropped_samples = self.tracker.dropped_samples_delta(); + let sample_range = self.tracker.contiguous_sample_range(); + let valid = self.integrity.is_clean() + && self.truncated_bytes == 0 + && self.tracker.malformed_sample_frames == 0 + && self.tracker.sample_segments <= 1; + Ok(PdqReadSummary { + frames_read: self.frames_read, + sample_frames_read: self.tracker.sample_frames, + samples_read: self.tracker.samples, + bytes_read: self.bytes_read, + file_crc32: self.running_crc.finalize(), + file_sha256: self.running_sha256.finalize(), + sample_range, + sample_rate_hz: self.tracker.uniform_sample_rate(), + sample_segments: self.tracker.sample_segments, + malformed_sample_frames: self.tracker.malformed_sample_frames, + truncated_bytes: self.truncated_bytes, + integrity: self.integrity, + valid, + }) + } +} + +pub fn inspect_pdq(path: impl AsRef) -> io::Result { + PdqReader::open(path)?.finish() +} + +#[derive(Default)] +struct FrameTracker { + previous_frame_sequence: Option, + frame_sequence_gaps: u64, + sample_frames: u64, + samples: u64, + first_sample_index: Option, + last_sample_end: Option, + previous_sample_end: Option, + first_sample_rate_hz: Option, + previous_sample_rate_hz: Option, + sample_rate_changed: bool, + sample_segments: u64, + malformed_sample_frames: u64, + first_dropped_samples: Option, + last_dropped_samples: Option, +} + +impl FrameTracker { + fn observe(&mut self, frame: &Frame) { + if let Some(previous) = self.previous_frame_sequence { + if frame.header.sequence != previous.wrapping_add(1) { + self.frame_sequence_gaps += 1; + } + } + self.previous_frame_sequence = Some(frame.header.sequence); + self.first_dropped_samples + .get_or_insert(frame.header.dropped_samples); + self.last_dropped_samples = Some(frame.header.dropped_samples); + + if frame.header.frame_type != FrameType::SamplesU16 { + return; + } + if !frame.payload.len().is_multiple_of(2) { + self.malformed_sample_frames += 1; + return; + } + let count = (frame.payload.len() / 2) as u64; + if count == 0 { + return; + } + let first = frame.header.first_sample_index; + let end = first.saturating_add(count); + let starts_new_segment = self.previous_sample_end.is_none() + || self.previous_sample_end != Some(first) + || self.previous_sample_rate_hz != Some(frame.header.sample_rate_hz); + if starts_new_segment { + self.sample_segments += 1; + } + if self + .first_sample_rate_hz + .is_some_and(|rate| rate != frame.header.sample_rate_hz) + { + self.sample_rate_changed = true; + } + self.first_sample_rate_hz + .get_or_insert(frame.header.sample_rate_hz); + self.previous_sample_rate_hz = Some(frame.header.sample_rate_hz); + self.first_sample_index.get_or_insert(first); + self.last_sample_end = Some(end); + self.previous_sample_end = Some(end); + self.sample_frames += 1; + self.samples += count; + } + + fn dropped_samples_delta(&self) -> u64 { + match (self.first_dropped_samples, self.last_dropped_samples) { + (Some(first), Some(last)) => u64::from(last.saturating_sub(first)), + _ => 0, + } + } + + fn uniform_sample_rate(&self) -> Option { + (!self.sample_rate_changed) + .then_some(self.first_sample_rate_hz) + .flatten() + } + + fn contiguous_sample_range(&self) -> Option { + if self.sample_segments != 1 || self.malformed_sample_frames > 0 { + return None; + } + Some(PdqSampleRange { + first_sample_index: self.first_sample_index?, + end_sample_index_exclusive: self.last_sample_end?, + sample_count: self.samples, }) } } @@ -73,9 +389,10 @@ impl PdqWriter { #[cfg(test)] mod tests { use super::*; - use crate::wire::{FrameHeader, FrameType, PROTOCOL_VERSION}; + use crate::wire::{FrameHeader, PROTOCOL_VERSION}; + use std::io::Cursor; - fn frame(sequence: u32) -> Frame { + fn control_frame(sequence: u32) -> Frame { Frame::build( FrameHeader { version: PROTOCOL_VERSION, @@ -92,50 +409,139 @@ mod tests { ) } - #[test] - fn writes_frames_verbatim_and_reports_validity() { + fn sample_frame(sequence: u32, first_index: u64, rate_hz: u32, codes: &[u16]) -> Frame { + Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::SamplesU16, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: first_index, + sample_rate_hz: rate_hz, + dropped_samples: 0, + crc32: 0, + }, + codes.iter().flat_map(|code| code.to_le_bytes()).collect(), + ) + } + + fn temp_dir(tag: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( - "stage-a-io-pdq-{}", + "stage-a-io-pdq-{tag}-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .unwrap() + .expect("clock") .as_nanos() )); - let path = dir.join("run.pdq"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + #[test] + fn writer_and_reader_agree_on_digest_size_and_sample_range() { + let dir = temp_dir("receipt"); + let path = dir.join("run.pdq"); + let frames = [ + sample_frame(10, 1_000, 20_000, &[1, 2, 3]), + sample_frame(11, 1_003, 20_000, &[4, 5]), + ]; let mut writer = PdqWriter::create(&path).expect("create pdq"); - let first = frame(1); - let second = frame(2); - writer.write_frame(&first).expect("write"); - writer.write_frame(&second).expect("write"); - let summary = writer + for frame in &frames { + writer.write_frame(frame).expect("write frame"); + } + let written = writer .finish(StreamIntegrity::default()) - .expect("finish pdq"); + .expect("finish writer"); + let read = inspect_pdq(&path).expect("inspect pdq"); - assert!(summary.valid); - assert_eq!(summary.frames_written, 2); - let on_disk = std::fs::read(&path).expect("read back"); - let mut expected = first.to_bytes(); - expected.extend_from_slice(&second.to_bytes()); - assert_eq!(on_disk, expected); - assert_eq!(summary.file_crc32, crate::wire::crc32(&expected)); + assert!(written.valid && read.valid); + assert_eq!(written.frames_written, 2); + assert_eq!(written.sample_frames_written, 2); + assert_eq!(written.samples_written, 5); + assert_eq!(written.bytes_written, read.bytes_read); + assert_eq!(written.file_crc32, read.file_crc32); + assert_eq!(written.file_sha256, read.file_sha256); + assert_eq!(written.file_sha256_hex().len(), 64); + assert_eq!( + written.sample_range, + Some(PdqSampleRange { + first_sample_index: 1_000, + end_sample_index_exclusive: 1_005, + sample_count: 5, + }) + ); + assert_eq!(written.sample_range, read.sample_range); + assert_eq!(written.sample_rate_hz, Some(20_000)); std::fs::remove_dir_all(dir).expect("cleanup"); } #[test] - fn integrity_faults_invalidate_the_run_but_keep_the_file() { - let dir = std::env::temp_dir().join(format!( - "stage-a-io-pdq-invalid-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let path = dir.join("run.pdq"); + fn reader_streams_frames_and_reports_crc_corruption() { + let first = control_frame(1).to_bytes(); + let mut corrupt = control_frame(2).to_bytes(); + let last = corrupt.len() - 1; + corrupt[last] ^= 0x80; + let third = control_frame(3).to_bytes(); + let bytes: Vec = first.into_iter().chain(corrupt).chain(third).collect(); + let mut reader = PdqReader::new(Cursor::new(bytes)); + let mut frames = Vec::new(); + let mut saw_corruption = false; + while let Some(event) = reader.next_event().expect("read event") { + match event { + PdqReadEvent::Frame(frame) => frames.push(frame.header.sequence), + PdqReadEvent::Corruption { crc_failures, .. } => { + saw_corruption |= crc_failures > 0; + } + PdqReadEvent::TruncatedTail { .. } => {} + } + } + assert_eq!(frames, [1, 3]); + assert!(saw_corruption); + let summary = reader.finish().expect("finish after iteration"); + assert!(!summary.valid); + assert_eq!(summary.integrity.crc_failures, 1); + assert_eq!(summary.integrity.sequence_gaps, 1); + } + + #[test] + fn truncated_tail_is_visible_and_invalid() { + let mut bytes = sample_frame(1, 0, 20_000, &[1, 2, 3]).to_bytes(); + bytes.extend_from_slice(b"PDA"); + let mut reader = PdqReader::new(Cursor::new(bytes)); + let mut truncated = 0; + while let Some(event) = reader.next_event().expect("read") { + if let PdqReadEvent::TruncatedTail { bytes } = event { + truncated += bytes; + } + } + assert_eq!(truncated, 3); + let summary = reader.finish().expect("finish"); + assert_eq!(summary.truncated_bytes, 3); + assert!(!summary.valid); + } + #[test] + fn discontinuous_samples_have_no_contiguous_range() { + let first = sample_frame(4, 100, 20_000, &[1, 2]).to_bytes(); + let second = sample_frame(5, 900, 50_000, &[3, 4]).to_bytes(); + let bytes: Vec = first.into_iter().chain(second).collect(); + let summary = PdqReader::new(Cursor::new(bytes)) + .finish() + .expect("inspect"); + assert_eq!(summary.sample_segments, 2); + assert_eq!(summary.sample_range, None); + assert_eq!(summary.sample_rate_hz, None); + assert!(!summary.valid); + } + + #[test] + fn explicit_integrity_faults_invalidate_writer_but_keep_the_file() { + let dir = temp_dir("invalid"); + let path = dir.join("run.pdq"); let mut writer = PdqWriter::create(&path).expect("create pdq"); - writer.write_frame(&frame(1)).expect("write"); + writer.write_frame(&control_frame(1)).expect("write frame"); let summary = writer .finish(StreamIntegrity { dropped_samples: 5, diff --git a/stage-a-io/src/sha256.rs b/stage-a-io/src/sha256.rs new file mode 100644 index 0000000..8447d31 --- /dev/null +++ b/stage-a-io/src/sha256.rs @@ -0,0 +1,259 @@ +//! Small streaming SHA-256 implementation used to finalize PDQ evidence. +//! +//! Keeping this implementation local avoids adding a crypto dependency to +//! the hardware-facing crate. It implements only unkeyed SHA-256 and is +//! tested against the FIPS 180-4 example vectors. + +use std::fmt; + +const INITIAL_STATE: [u32; 8] = [ + 0x6a09_e667, + 0xbb67_ae85, + 0x3c6e_f372, + 0xa54f_f53a, + 0x510e_527f, + 0x9b05_688c, + 0x1f83_d9ab, + 0x5be0_cd19, +]; + +const ROUND_CONSTANTS: [u32; 64] = [ + 0x428a_2f98, + 0x7137_4491, + 0xb5c0_fbcf, + 0xe9b5_dba5, + 0x3956_c25b, + 0x59f1_11f1, + 0x923f_82a4, + 0xab1c_5ed5, + 0xd807_aa98, + 0x1283_5b01, + 0x2431_85be, + 0x550c_7dc3, + 0x72be_5d74, + 0x80de_b1fe, + 0x9bdc_06a7, + 0xc19b_f174, + 0xe49b_69c1, + 0xefbe_4786, + 0x0fc1_9dc6, + 0x240c_a1cc, + 0x2de9_2c6f, + 0x4a74_84aa, + 0x5cb0_a9dc, + 0x76f9_88da, + 0x983e_5152, + 0xa831_c66d, + 0xb003_27c8, + 0xbf59_7fc7, + 0xc6e0_0bf3, + 0xd5a7_9147, + 0x06ca_6351, + 0x1429_2967, + 0x27b7_0a85, + 0x2e1b_2138, + 0x4d2c_6dfc, + 0x5338_0d13, + 0x650a_7354, + 0x766a_0abb, + 0x81c2_c92e, + 0x9272_2c85, + 0xa2bf_e8a1, + 0xa81a_664b, + 0xc24b_8b70, + 0xc76c_51a3, + 0xd192_e819, + 0xd699_0624, + 0xf40e_3585, + 0x106a_a070, + 0x19a4_c116, + 0x1e37_6c08, + 0x2748_774c, + 0x34b0_bcb5, + 0x391c_0cb3, + 0x4ed8_aa4a, + 0x5b9c_ca4f, + 0x682e_6ff3, + 0x748f_82ee, + 0x78a5_636f, + 0x84c8_7814, + 0x8cc7_0208, + 0x90be_fffa, + 0xa450_6ceb, + 0xbef9_a3f7, + 0xc671_78f2, +]; + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Sha256Digest([u8; 32]); + +impl Sha256Digest { + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub fn to_hex(self) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(64); + for byte in self.0 { + output.push(char::from(HEX[usize::from(byte >> 4)])); + output.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + output + } +} + +impl fmt::Display for Sha256Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.to_hex()) + } +} + +impl fmt::Debug for Sha256Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Sha256Digest({self})") + } +} + +pub(crate) struct Sha256 { + state: [u32; 8], + buffer: [u8; 64], + buffer_len: usize, + bytes_seen: u64, +} + +impl Default for Sha256 { + fn default() -> Self { + Self { + state: INITIAL_STATE, + buffer: [0; 64], + buffer_len: 0, + bytes_seen: 0, + } + } +} + +impl Sha256 { + pub(crate) fn update(&mut self, mut bytes: &[u8]) { + self.bytes_seen = self.bytes_seen.wrapping_add(bytes.len() as u64); + if self.buffer_len > 0 { + let fill = (64 - self.buffer_len).min(bytes.len()); + self.buffer[self.buffer_len..self.buffer_len + fill].copy_from_slice(&bytes[..fill]); + self.buffer_len += fill; + bytes = &bytes[fill..]; + if self.buffer_len == 64 { + let block = self.buffer; + self.compress(&block); + self.buffer_len = 0; + } else { + return; + } + } + + while bytes.len() >= 64 { + let block: &[u8; 64] = bytes[..64].try_into().expect("exact SHA-256 block"); + self.compress(block); + bytes = &bytes[64..]; + } + self.buffer[..bytes.len()].copy_from_slice(bytes); + self.buffer_len = bytes.len(); + } + + pub(crate) fn finalize(mut self) -> Sha256Digest { + let bit_len = self.bytes_seen.wrapping_mul(8); + self.buffer[self.buffer_len] = 0x80; + self.buffer_len += 1; + if self.buffer_len > 56 { + self.buffer[self.buffer_len..].fill(0); + let block = self.buffer; + self.compress(&block); + self.buffer = [0; 64]; + } else { + self.buffer[self.buffer_len..56].fill(0); + } + self.buffer[56..64].copy_from_slice(&bit_len.to_be_bytes()); + let block = self.buffer; + self.compress(&block); + + let mut digest = [0; 32]; + for (chunk, word) in digest.chunks_exact_mut(4).zip(self.state) { + chunk.copy_from_slice(&word.to_be_bytes()); + } + Sha256Digest(digest) + } + + fn compress(&mut self, block: &[u8; 64]) { + let mut schedule = [0_u32; 64]; + for (word, bytes) in schedule[..16].iter_mut().zip(block.chunks_exact(4)) { + *word = u32::from_be_bytes(bytes.try_into().expect("four-byte word")); + } + for index in 16..64 { + let x = schedule[index - 15]; + let y = schedule[index - 2]; + let sigma0 = x.rotate_right(7) ^ x.rotate_right(18) ^ (x >> 3); + let sigma1 = y.rotate_right(17) ^ y.rotate_right(19) ^ (y >> 10); + schedule[index] = schedule[index - 16] + .wrapping_add(sigma0) + .wrapping_add(schedule[index - 7]) + .wrapping_add(sigma1); + } + + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = self.state; + for (word, constant) in schedule.into_iter().zip(ROUND_CONSTANTS) { + let sum1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let choose = (e & f) ^ (!e & g); + let temp1 = h + .wrapping_add(sum1) + .wrapping_add(choose) + .wrapping_add(constant) + .wrapping_add(word); + let sum0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let majority = (a & b) ^ (a & c) ^ (b & c); + let temp2 = sum0.wrapping_add(majority); + h = g; + g = f; + f = e; + e = d.wrapping_add(temp1); + d = c; + c = b; + b = a; + a = temp1.wrapping_add(temp2); + } + for (state, value) in self.state.iter_mut().zip([a, b, c, d, e, f, g, h]) { + *state = state.wrapping_add(value); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digest(chunks: &[&[u8]]) -> String { + let mut sha = Sha256::default(); + for chunk in chunks { + sha.update(chunk); + } + sha.finalize().to_hex() + } + + #[test] + fn matches_fips_vectors_and_fragmentation() { + assert_eq!( + digest(&[b""]), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + digest(&[b"a", b"b", b"c"]), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + digest(&[b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"]), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + } +} diff --git a/stage-a-io/src/sidecar.rs b/stage-a-io/src/sidecar.rs index 0b591bc..0ebd036 100644 --- a/stage-a-io/src/sidecar.rs +++ b/stage-a-io/src/sidecar.rs @@ -191,8 +191,18 @@ mod tests { let pdq = PdqSummary { path: PathBuf::from("/data/A1-20260713-01.pdq"), frames_written: 128, + sample_frames_written: 128, + samples_written: 32_768, bytes_written: 65_536, file_crc32: 0xDEAD_BEEF, + file_sha256: crate::Sha256Digest::from_bytes([0xAB; 32]), + sample_range: Some(crate::PdqSampleRange { + first_sample_index: 0, + end_sample_index_exclusive: 32_768, + sample_count: 32_768, + }), + sample_rate_hz: Some(20_000), + sample_segments: 1, integrity: StreamIntegrity::default(), valid: true, }; diff --git a/stage-a-io/src/wire.rs b/stage-a-io/src/wire.rs index c42a249..bb90b5a 100644 --- a/stage-a-io/src/wire.rs +++ b/stage-a-io/src/wire.rs @@ -139,7 +139,8 @@ impl Frame { /// Decodes the payload of a `SamplesU16` frame into ADC codes. pub fn samples(&self) -> Option> { - if self.header.frame_type != FrameType::SamplesU16 || self.payload.len() % 2 != 0 { + if self.header.frame_type != FrameType::SamplesU16 || !self.payload.len().is_multiple_of(2) + { return None; } Some( @@ -157,6 +158,21 @@ impl Frame { } std::str::from_utf8(&self.payload).ok() } + + /// Decodes the payload of a `Marker` frame (phase-0 fiducial on the device + /// clock). + pub fn marker(&self) -> Option { + if self.header.frame_type != FrameType::Marker || self.payload.len() != 16 { + return None; + } + let p = &self.payload; + Some(MarkerPayload { + sample_index: u64::from_le_bytes(p[0..8].try_into().ok()?), + tick_us: u32::from_le_bytes(p[8..12].try_into().ok()?), + level: p[12], + source: p[13], + }) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -189,6 +205,32 @@ impl SummaryPayload { } } +/// A `Marker` frame payload: a phase-0 fiducial stamped on the device clock +/// (matches the firmware `MarkerPayload`; `source = 1` is a modulation phase-0). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MarkerPayload { + /// ADC sample index at the fiducial — aligns the marker with the stream. + pub sample_index: u64, + pub tick_us: u32, + pub level: u8, + pub source: u8, +} + +/// `source` value the firmware stamps on a modulation phase-0 marker. +pub const MARKER_SOURCE_PHASE0: u8 = 1; + +impl MarkerPayload { + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(16); + out.extend_from_slice(&self.sample_index.to_le_bytes()); + out.extend_from_slice(&self.tick_us.to_le_bytes()); + out.push(self.level); + out.push(self.source); + out.extend_from_slice(&[0_u8, 0_u8]); // reserved[2] + out + } +} + /// CRC32 (IEEE, reflected, init/final 0xFFFF_FFFF) — identical to the /// firmware's `crc32Update` loop. pub fn crc32(data: &[u8]) -> u32 { @@ -270,6 +312,22 @@ impl FrameParser { self.buffer.extend_from_slice(bytes); } + /// Bytes retained while waiting for a complete header or payload. + /// Primarily useful at a finite-file EOF, where a nonzero value means + /// the PDQ ends with a truncated frame or garbage tail. + pub fn buffered_len(&self) -> usize { + self.buffer.len() + } + + /// Discards and returns the number of bytes still buffered. Live serial + /// readers normally never need this; finite-file readers use it once at + /// EOF to report a truncated tail without exposing parser internals. + pub fn discard_buffered(&mut self) -> usize { + let len = self.buffer.len(); + self.buffer.clear(); + len + } + pub fn next_event(&mut self) -> Option { loop { // Scan to the next plausible magic. @@ -443,6 +501,33 @@ mod tests { assert!((summary.mean_code() - 1953.125).abs() < 1e-9); } + #[test] + fn marker_payload_round_trips() { + let marker = MarkerPayload { + sample_index: 1_234_567, + tick_us: 987_654, + level: 1, + source: MARKER_SOURCE_PHASE0, + }; + let frame = Frame::build( + FrameHeader { + version: PROTOCOL_VERSION, + frame_type: FrameType::Marker, + flags: 0, + sequence: 9, + payload_bytes: 0, + first_sample_index: 0, + sample_rate_hz: 20_000, + dropped_samples: 0, + crc32: 0, + }, + marker.encode(), + ); + assert_eq!(frame.marker(), Some(marker)); + // A samples decode must not accept a marker frame. + assert_eq!(frame.samples(), None); + } + #[test] fn samples_frame_decodes_codes() { let codes = [1_u16, 2, 4_095]; diff --git a/stage-a-plugin-contract/Cargo.toml b/stage-a-plugin-contract/Cargo.toml new file mode 100644 index 0000000..9446f85 --- /dev/null +++ b/stage-a-plugin-contract/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "stage-a-plugin-contract" +version = "0.1.0" +edition = "2021" +license = "MIT" +authors = ["Mika Uthmann "] +description = "Serde-only inter-plugin control and status contract for the Stage-A device-owner plugins" + +[dependencies] +serde = { version = "1", features = ["derive"] } + +[dev-dependencies] +serde_json = "1" + +[lints.rust] +unsafe_code = "forbid" + diff --git a/stage-a-plugin-contract/README.md b/stage-a-plugin-contract/README.md new file mode 100644 index 0000000..735434c --- /dev/null +++ b/stage-a-plugin-contract/README.md @@ -0,0 +1,47 @@ +# Stage-A Plugin Contract + +This crate is the serde-only control-plane contract between the Stage-A workflow plugins and the +two plugins that permanently own the Teensy ports: + +- `stage-a-modulation` owns and controls the command port; +- `stage-a-photodiode` owns and reads the PDA1 stream port; +- experiment plugins such as `stage-a-a1` orchestrate those owners without opening either port. + +The crate deliberately has no Augur, serial, filesystem, or thread dependency. Its payloads can be +serialized through the host's persistent plugin context. Every context key and payload is +explicitly versioned. A request carries a unique request ID, the target owner instance, an +optional lease and run ID, and an optional requested semantic revision. Responses echo those +identities and report the ACKed revision. + +## Mailboxes + +| Direction | Context key | +|---|---| +| orchestrator → modulation owner | `stage_a.modulation_request.v1` | +| modulation owner → orchestrator | `stage_a.modulation_response.v1` | +| modulation owner snapshot | `stage_a.modulation_state.v1` | +| orchestrator → photodiode owner | `stage_a.photodiode_request.v1` | +| photodiode owner → orchestrator | `stage_a.photodiode_response.v1` | +| photodiode owner snapshot | `stage_a.photodiode_summary.v1` | + +Persistent context is a last-writer-wins mailbox, not a queue. An orchestrator must keep at most +one outstanding request per owner, retain it until its request ID is acknowledged, and never +reuse a request ID. Owners must make duplicate delivery idempotent by returning the original +result without repeating the effect. + +## Safety and data boundaries + +Control commands are semantic (`PrepareA1`, `SafeOff`, `BeginRecording`, and so on), not raw +firmware strings or remote setting changes. Automated mutations require an owner-issued lease; +leases expire unless renewed. Owner snapshots carry an instance ID and freshness deadline so an +orchestrator can detect reloads and stale state. + +Photodiode messages contain only bounded summaries and named PDQ receipts. Raw ADC arrays never +cross the JSON context. The finalized receipt names the PDQ/sidecar, SHA-256, byte and frame +counts, contiguous sample range, stream integrity, and validity. Analysis reads the finalized PDQ +through `stage-a-io`. + +`SynchronizationV1::Unsynced` is a first-class state. Missing firmware configuration revisions, +owner restarts, stream-epoch changes, stale snapshots, or run-ID mismatches must be reported as +UNSYNCED rather than inferred away. + diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs new file mode 100644 index 0000000..e7e1638 --- /dev/null +++ b/stage-a-plugin-contract/src/lib.rs @@ -0,0 +1,853 @@ +//! Versioned, serde-only messages shared by Stage-A experiment workflows and +//! the two persistent Teensy device-owner plugins. +//! +//! This crate contains semantic control-plane types only. It intentionally +//! contains no Augur ABI types, serial transports, filesystem access, raw ADC +//! arrays, or experiment state machines. + +#![forbid(unsafe_code)] + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; + +pub const CONTRACT_VERSION_V1: u16 = 1; + +pub const PLUGIN_ID_STAGE_A_MODULATION: &str = "stage-a.modulation"; +pub const PLUGIN_ID_STAGE_A_PHOTODIODE: &str = "stage-a.photodiode"; +pub const SERVICE_STAGE_A_MODULATION_CONTROL_V1: &str = "stage_a.modulation.control.v1"; +pub const SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1: &str = "stage_a.photodiode.control.v1"; + +pub const CTX_STAGE_A_MODULATION_REQUEST_V1: &str = "stage_a.modulation_request.v1"; +pub const CTX_STAGE_A_MODULATION_RESPONSE_V1: &str = "stage_a.modulation_response.v1"; +pub const CTX_STAGE_A_MODULATION_STATE_V1: &str = "stage_a.modulation_state.v1"; +pub const CTX_STAGE_A_PHOTODIODE_REQUEST_V1: &str = "stage_a.photodiode_request.v1"; +pub const CTX_STAGE_A_PHOTODIODE_RESPONSE_V1: &str = "stage_a.photodiode_response.v1"; +pub const CTX_STAGE_A_PHOTODIODE_SUMMARY_V1: &str = "stage_a.photodiode_summary.v1"; + +macro_rules! string_id { + ($name:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] + #[serde(transparent)] + pub struct $name(pub String); + + impl $name { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl From<&str> for $name { + fn from(value: &str) -> Self { + Self(value.to_owned()) + } + } + + impl From for $name { + fn from(value: String) -> Self { + Self(value) + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } + } + }; +} + +string_id!(ClientId); +string_id!(LeaseId); +string_id!(OwnerInstanceId); +string_id!(RunId); + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default, +)] +#[serde(transparent)] +pub struct RequestId(pub u64); + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default, +)] +#[serde(transparent)] +pub struct SemanticRevision(pub u64); + +/// Wall-clock freshness information transferable between dynamic plugins. +/// The consumer determines staleness against its current Unix time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct FreshnessV1 { + pub observed_at_unix_ms: u64, + pub valid_for_ms: u64, +} + +impl FreshnessV1 { + pub fn is_stale_at(self, now_unix_ms: u64) -> bool { + now_unix_ms.saturating_sub(self.observed_at_unix_ms) > self.valid_for_ms + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum ConnectionStateV1 { + Disconnected, + Connecting, + Connected { + port_label: String, + firmware_version: Option, + }, + Faulted { + message: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaseSnapshotV1 { + pub lease_id: LeaseId, + pub holder: ClientId, + pub expires_at_unix_ms: u64, + pub run_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UnsyncedReasonV1 { + NoOwnerSnapshot, + OwnerRestarted, + StaleSnapshot, + NoLease, + LeaseMismatch, + RunMismatch, + RequestedRevisionNotAcknowledged, + FirmwareRevisionUnavailable, + StreamEpochChanged, + DeviceFault, + Other(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum SynchronizationV1 { + Synced { + run_id: RunId, + acknowledged_revision: SemanticRevision, + stream_epoch: Option, + }, + Unsynced { + reason: UnsyncedReasonV1, + detail: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ServiceErrorCodeV1 { + ContractVersion, + WrongOwnerInstance, + StaleRequest, + DuplicateRequestConflict, + NotConnected, + LeaseRequired, + LeaseBusy, + LeaseMismatch, + LeaseExpired, + UnsafeExecutionContext, + InvalidCommand, + InvalidPath, + DeviceRejected, + Transport, + Io, + Integrity, + Internal, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ServiceErrorV1 { + pub code: ServiceErrorCodeV1, + pub message: String, + pub retryable: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RequestOutcomeV1 { + InProgress, + Applied, + Rejected, +} + +/// Common request envelope. The command-specific aliases below are the +/// public mailbox payloads. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RequestEnvelopeV1 { + pub contract_version: u16, + pub request_id: RequestId, + pub requester: ClientId, + /// `None` is allowed only for discovery/connect or first lease acquire. + pub target_owner_instance: Option, + pub lease_id: Option, + pub run_id: Option, + pub requested_revision: Option, + pub issued_at_unix_ms: u64, + pub command: C, +} + +impl RequestEnvelopeV1 { + pub fn new(request_id: RequestId, requester: ClientId, command: C) -> Self { + Self { + contract_version: CONTRACT_VERSION_V1, + request_id, + requester, + target_owner_instance: None, + lease_id: None, + run_id: None, + requested_revision: None, + issued_at_unix_ms: 0, + command, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseCommonV1 { + pub contract_version: u16, + pub request_id: RequestId, + pub owner_instance: OwnerInstanceId, + pub run_id: Option, + pub requested_revision: Option, + pub acknowledged_revision: Option, + pub outcome: RequestOutcomeV1, + pub completed_at_unix_ms: Option, + pub error: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PeriodicWaveformV1 { + Sine, + Square, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum WaveformV1 { + Off, + Constant { + level_dac: u16, + }, + Periodic { + waveform: PeriodicWaveformV1, + min_dac: u16, + max_dac: u16, + frequency_millihz: u64, + }, +} + +/// Complete semantic configuration for one A1 controller acquisition. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct A1AcquisitionConfigV1 { + pub waveform: PeriodicWaveformV1, + pub frequency_millihz: u64, + pub center_dac: u16, + pub amplitude_dac: u16, + pub sample_rate_hz: u32, + pub block_samples: u32, + pub emit_raw_samples: bool, + pub emit_summary: bool, + pub optical_lut_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ModulationCommandV1 { + Connect, + Disconnect { + safe_off: bool, + reason: String, + }, + AcquireLease { + ttl_ms: u64, + }, + RenewLease { + ttl_ms: u64, + }, + ReleaseLease { + safe_off: bool, + reason: String, + }, + SetWaveform { + waveform: WaveformV1, + }, + /// Retarget the owner's *calibrated optical drive* to a new modulation + /// depth `a` (log contrast, in milli-units) without changing anything else + /// about the armed drive: waveform shape, frequency, operating point and + /// calibration stay whatever the operator armed in the modulation plugin. + /// This is the scoped amplitude-sweep path (A1 automation): the owner + /// rejects the command when its current drive cannot express `a` + /// (manual DAC method or constant mode) or the device link is closed. + SetOpticalDepth { + depth_a_milli: u32, + }, + PrepareA1 { + configuration: A1AcquisitionConfigV1, + }, + StartAcquisition, + StopAcquisition { + reason: String, + }, + SafeOff { + reason: String, + }, +} + +pub type ModulationRequestV1 = RequestEnvelopeV1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ControllerStateV1 { + Unknown, + SafeIdle, + Configured, + Running, + Faulted, +} + +/// The full desired or board-acknowledged command-port state at one semantic +/// revision. Owners never infer an ACK from the requested state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModulationTargetV1 { + pub revision: SemanticRevision, + pub waveform: Option, + pub a1_configuration: Option, + pub acquisition_running: bool, + pub board_dac_code: Option, + pub firmware_configuration_revision: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ModulationResponseV1 { + #[serde(flatten)] + pub common: ResponseCommonV1, + pub controller_state: ControllerStateV1, + pub acknowledged_target: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ModulationStateV1 { + pub contract_version: u16, + pub owner_instance: OwnerInstanceId, + pub service_revision: u64, + pub connection: ConnectionStateV1, + pub capabilities: Vec, + pub lease: Option, + pub controller_state: ControllerStateV1, + pub active_run_id: Option, + pub requested: Option, + pub acknowledged: Option, + pub synchronization: SynchronizationV1, + pub last_response: Option, + pub freshness: FreshnessV1, + /// Identifier of the measured Pockels transfer calibration currently + /// applied to `V_null`/`Vπ`, so a consumer's sidecar can cite which + /// inversion produced a run's optical depth. `None` means the operator + /// entered the lobe parameters by hand. Additive in V1. + #[serde(default)] + pub calibration_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct StreamIntegrityV1 { + pub skipped_bytes: u64, + pub crc_failures: u64, + pub sequence_gaps: u64, + pub dropped_samples: u64, + pub segment_restarts: u64, + pub truncated_bytes: u64, +} + +impl StreamIntegrityV1 { + pub fn is_clean(self) -> bool { + self == Self::default() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct SampleRangeV1 { + pub first_sample_index: u64, + pub end_sample_index_exclusive: u64, + pub sample_count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct Sha256V1(String); + +impl Sha256V1 { + pub fn parse(value: impl Into) -> Result { + let value = value.into(); + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err("SHA-256 must be exactly 64 hexadecimal characters".into()); + } + Ok(Self(value.to_ascii_lowercase())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for Sha256V1 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for Sha256V1 { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::parse(value).map_err(serde::de::Error::custom) + } +} + +/// The exact file to open at a recording boundary. Metadata is deliberately +/// string-valued and bounded by the owner; scientific sidecars remain the +/// canonical rich metadata record. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PdqStartSpecV1 { + pub pdq_path: String, + pub sidecar_path: String, + pub expected_sample_rate_hz: Option, + pub expected_stream_epoch: Option, + pub metadata: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PdqStartedReceiptV1 { + pub run_id: RunId, + pub pdq_path: String, + pub sidecar_path: String, + pub opened_at_unix_ms: u64, + pub stream_epoch: u64, + pub first_sample_index: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PdqTerminationV1 { + Completed, + OperatorStopped, + LeaseExpired, + SafeOff, + DeviceFault, + IoFault, + Aborted, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PdqFinalizedReceiptV1 { + pub run_id: RunId, + pub pdq_path: String, + pub sidecar_path: String, + pub opened_at_unix_ms: u64, + pub finalized_at_unix_ms: u64, + pub file_size_bytes: u64, + pub sha256: Sha256V1, + pub frames_written: u64, + pub sample_frames_written: u64, + pub sample_range: Option, + pub sample_rate_hz: Option, + pub segment_count: u64, + pub integrity: StreamIntegrityV1, + pub termination: PdqTerminationV1, + pub valid: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PdqReceiptV1 { + Started(PdqStartedReceiptV1), + Finalized(PdqFinalizedReceiptV1), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum PhotodiodeCommandV1 { + Connect, + Disconnect { + finalize_recording: bool, + reason: String, + }, + AcquireLease { + ttl_ms: u64, + }, + RenewLease { + ttl_ms: u64, + }, + ReleaseLease { + finalize_recording: bool, + reason: String, + }, + BeginRecording { + specification: PdqStartSpecV1, + }, + FinalizeRecording { + termination: PdqTerminationV1, + }, + AbortRecording { + reason: String, + }, +} + +pub type PhotodiodeRequestV1 = RequestEnvelopeV1; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeResponseV1 { + #[serde(flatten)] + pub common: ResponseCommonV1, + pub receipt: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeCalibrationV1 { + pub adc_calibration_id: String, + pub dark_id: String, + pub anchor_id: String, + pub dark_volts: f64, + /// Named full-extinction anchor after dark subtraction. + pub total_power_volts: f64, +} + +/// Bounded optical result for one named run. It contains no raw or decimated +/// waveform samples; the finalized PDQ remains the source for replay. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeOpticalSummaryV1 { + pub run_id: RunId, + pub calibration: PhotodiodeCalibrationV1, + pub measured_log_contrast: f64, + pub log_contrast_stddev: Option, + pub excitation_min_volts: f64, + pub excitation_max_volts: f64, + pub excitation_headroom_volts: f64, + pub low_clip_fraction: f64, + pub high_clip_fraction: f64, + pub measured_frequency_hz: Option, + pub fundamental_phase_rad: Option, + pub total_harmonic_distortion: Option, +} + +/// Settled detector level over the newest averaging window, in **raw detector +/// volts**: the ADC affine map only, before dark subtraction and before any +/// [`PhotodiodeOpticalSummaryV1`] geometry transform. Unlike the optical +/// summary this never refuses — it stays present while the window clips (see +/// `clipped`), because a consumer sweeping a static transfer curve needs a +/// level exactly where the detector is brightest. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeLevelV1 { + pub mean_volts: f64, + /// Spread over the averaged window. A settled `CONST` point has a small + /// peak-to-peak; a drifting or still-slewing one does not. + pub peak_to_peak_volts: f64, + pub sample_count: u64, + /// Exclusive end of the averaged window on the device sample clock. The + /// window covers `[end_sample_index - sample_count, end_sample_index)`, so + /// a consumer can prove a level was measured *after* it commanded a + /// change without needing a shared wall clock. + pub end_sample_index: u64, + /// The window touches an ADC rail; `mean_volts` is a truncated estimate. + pub clipped: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeStreamV1 { + pub stream_epoch: u64, + pub sample_range: Option, + pub sample_rate_hz: Option, + pub latest_adc_code: Option, + pub integrity: StreamIntegrityV1, + /// Additive in V1: absent from older owners, and older consumers ignore it. + #[serde(default)] + pub level: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PhotodiodeSummaryV1 { + pub contract_version: u16, + pub owner_instance: OwnerInstanceId, + pub service_revision: u64, + pub connection: ConnectionStateV1, + pub lease: Option, + pub active_run_id: Option, + pub requested_revision: Option, + pub acknowledged_revision: Option, + pub stream: PhotodiodeStreamV1, + pub active_recording: Option, + pub last_finalized_recording: Option, + pub optical_summary: Option, + pub synchronization: SynchronizationV1, + pub last_response: Option, + pub freshness: FreshnessV1, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn common(request_id: u64) -> ResponseCommonV1 { + ResponseCommonV1 { + contract_version: CONTRACT_VERSION_V1, + request_id: RequestId(request_id), + owner_instance: OwnerInstanceId::from("owner-7"), + run_id: Some(RunId::from("A1-20260721-003")), + requested_revision: Some(SemanticRevision(4)), + acknowledged_revision: Some(SemanticRevision(4)), + outcome: RequestOutcomeV1::Applied, + completed_at_unix_ms: Some(1_721_000_001_000), + error: None, + } + } + + #[test] + fn context_keys_are_stable_and_versioned() { + assert_eq!(PLUGIN_ID_STAGE_A_MODULATION, "stage-a.modulation"); + assert_eq!(PLUGIN_ID_STAGE_A_PHOTODIODE, "stage-a.photodiode"); + assert_eq!( + SERVICE_STAGE_A_MODULATION_CONTROL_V1, + "stage_a.modulation.control.v1" + ); + assert_eq!( + SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1, + "stage_a.photodiode.control.v1" + ); + assert_eq!( + CTX_STAGE_A_MODULATION_REQUEST_V1, + "stage_a.modulation_request.v1" + ); + assert_eq!( + CTX_STAGE_A_MODULATION_RESPONSE_V1, + "stage_a.modulation_response.v1" + ); + assert_eq!( + CTX_STAGE_A_MODULATION_STATE_V1, + "stage_a.modulation_state.v1" + ); + assert_eq!( + CTX_STAGE_A_PHOTODIODE_REQUEST_V1, + "stage_a.photodiode_request.v1" + ); + assert_eq!( + CTX_STAGE_A_PHOTODIODE_RESPONSE_V1, + "stage_a.photodiode_response.v1" + ); + assert_eq!( + CTX_STAGE_A_PHOTODIODE_SUMMARY_V1, + "stage_a.photodiode_summary.v1" + ); + } + + #[test] + fn modulation_request_round_trips_with_semantic_discriminants() { + let mut request = ModulationRequestV1::new( + RequestId(12), + ClientId::from("stage-a-a1"), + ModulationCommandV1::PrepareA1 { + configuration: A1AcquisitionConfigV1 { + waveform: PeriodicWaveformV1::Sine, + frequency_millihz: 10_000, + center_dac: 2_048, + amplitude_dac: 512, + sample_rate_hz: 20_000, + block_samples: 256, + emit_raw_samples: true, + emit_summary: true, + optical_lut_id: Some("lut-2026-07".into()), + }, + }, + ); + request.target_owner_instance = Some(OwnerInstanceId::from("mod-owner-1")); + request.lease_id = Some(LeaseId::from("lease-a1")); + request.run_id = Some(RunId::from("run-3")); + request.requested_revision = Some(SemanticRevision(9)); + request.issued_at_unix_ms = 42; + + let json = serde_json::to_value(&request).expect("serializes"); + assert_eq!(json["command"]["kind"], "prepare_a1"); + assert_eq!( + json["command"]["configuration"]["frequency_millihz"], + 10_000 + ); + let decoded: ModulationRequestV1 = serde_json::from_value(json).expect("deserializes"); + assert_eq!(decoded, request); + } + + #[test] + fn set_optical_depth_round_trips_in_milli_units() { + let request = ModulationRequestV1::new( + RequestId(7), + ClientId::from("stage-a-a1"), + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: 1_250, + }, + ); + let json = serde_json::to_value(&request).expect("serializes"); + assert_eq!(json["command"]["kind"], "set_optical_depth"); + assert_eq!(json["command"]["depth_a_milli"], 1_250); + let decoded: ModulationRequestV1 = serde_json::from_value(json).expect("deserializes"); + assert_eq!(decoded, request); + } + + #[test] + fn snapshots_keep_requested_and_acknowledged_revisions_distinct() { + let requested = ModulationTargetV1 { + revision: SemanticRevision(5), + waveform: Some(WaveformV1::Constant { level_dac: 900 }), + a1_configuration: None, + acquisition_running: false, + board_dac_code: None, + firmware_configuration_revision: None, + }; + let acknowledged = ModulationTargetV1 { + revision: SemanticRevision(4), + waveform: Some(WaveformV1::Constant { level_dac: 800 }), + board_dac_code: Some(800), + ..requested.clone() + }; + let snapshot = ModulationStateV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::from("mod-owner-1"), + service_revision: 17, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: Some("0.4.0".into()), + }, + capabilities: vec!["MOD".into(), "PDSTREAM".into()], + lease: None, + controller_state: ControllerStateV1::SafeIdle, + active_run_id: None, + requested: Some(requested), + acknowledged: Some(acknowledged), + synchronization: SynchronizationV1::Unsynced { + reason: UnsyncedReasonV1::RequestedRevisionNotAcknowledged, + detail: Some("requested 5, acknowledged 4".into()), + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: 100, + valid_for_ms: 500, + }, + calibration_id: Some("pockels-20260724-120000".into()), + }; + let encoded = serde_json::to_vec(&snapshot).expect("serializes"); + let decoded: ModulationStateV1 = serde_json::from_slice(&encoded).expect("deserializes"); + assert_eq!(decoded.requested.unwrap().revision, SemanticRevision(5)); + assert_eq!(decoded.acknowledged.unwrap().revision, SemanticRevision(4)); + assert!(matches!( + decoded.synchronization, + SynchronizationV1::Unsynced { .. } + )); + } + + #[test] + fn finalized_pdq_receipt_round_trips_without_raw_samples() { + let receipt = PdqFinalizedReceiptV1 { + run_id: RunId::from("run-3"), + pdq_path: "/data/run-3_pd.pdq".into(), + sidecar_path: "/data/run-3.toml".into(), + opened_at_unix_ms: 1_000, + finalized_at_unix_ms: 2_000, + file_size_bytes: 8_192, + sha256: Sha256V1::parse("ab".repeat(32)).expect("digest"), + frames_written: 32, + sample_frames_written: 30, + sample_range: Some(SampleRangeV1 { + first_sample_index: 10_000, + end_sample_index_exclusive: 17_680, + sample_count: 7_680, + }), + sample_rate_hz: Some(20_000), + segment_count: 1, + integrity: StreamIntegrityV1::default(), + termination: PdqTerminationV1::Completed, + valid: true, + }; + let response = PhotodiodeResponseV1 { + common: common(22), + receipt: Some(PdqReceiptV1::Finalized(receipt.clone())), + }; + let json = serde_json::to_value(&response).expect("serializes"); + assert_eq!(json["receipt"]["kind"], "finalized"); + assert!(json.to_string().len() < 2_048, "receipt stays bounded"); + let decoded: PhotodiodeResponseV1 = serde_json::from_value(json).expect("deserializes"); + assert_eq!(decoded.receipt, Some(PdqReceiptV1::Finalized(receipt))); + } + + #[test] + fn sha256_and_freshness_validate_boundaries() { + assert!(Sha256V1::parse("0".repeat(64)).is_ok()); + assert!( + Sha256V1::parse("A".repeat(64)).is_ok_and(|digest| digest.as_str() == "a".repeat(64)) + ); + assert!(Sha256V1::parse("0".repeat(63)).is_err()); + assert!(Sha256V1::parse("z".repeat(64)).is_err()); + assert!(serde_json::from_str::(&format!("\"{}\"", "z".repeat(64))).is_err()); + + let freshness = FreshnessV1 { + observed_at_unix_ms: 1_000, + valid_for_ms: 500, + }; + assert!(!freshness.is_stale_at(1_500)); + assert!(freshness.is_stale_at(1_501)); + assert!(!freshness.is_stale_at(900), "clock rollback saturates"); + } + + #[test] + fn stream_integrity_is_fail_closed() { + assert!(StreamIntegrityV1::default().is_clean()); + assert!(!StreamIntegrityV1 { + segment_restarts: 1, + ..StreamIntegrityV1::default() + } + .is_clean()); + } + + #[test] + fn additive_v1_fields_decode_from_payloads_that_predate_them() { + // An older owner's stream block carries no `level`, and an older + // modulation state no `calibration_id`. Both must still decode. + let stream: PhotodiodeStreamV1 = serde_json::from_value(json!({ + "stream_epoch": 3, + "sample_range": null, + "sample_rate_hz": 20_000, + "latest_adc_code": 1_024, + "integrity": StreamIntegrityV1::default(), + })) + .expect("stream without level decodes"); + assert!(stream.level.is_none()); + + let level = PhotodiodeLevelV1 { + mean_volts: 1.5, + peak_to_peak_volts: 0.01, + sample_count: 4_096, + end_sample_index: 1_000_000, + clipped: false, + }; + let round_tripped: PhotodiodeLevelV1 = + serde_json::from_value(serde_json::to_value(level).expect("serializes")) + .expect("deserializes"); + assert_eq!(round_tripped, level); + // The window is identified without a wall clock: it ends at + // `end_sample_index` and spans `sample_count` samples. + assert_eq!( + level.end_sample_index - level.sample_count, + 1_000_000 - 4_096 + ); + } +} From c0e091a1010ac24a8e251a6adc2d1a6a8dc8903e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:02:05 +0200 Subject: [PATCH 22/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20publish=20?= =?UTF-8?q?the=20excitation=20contrast=20independently=20of=20the=20displa?= =?UTF-8?q?y=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The photodiode sits behind the PBS reject port and measures the complement I_pd = I_tot - I_exc — a property of the bench, not of what the operator chose to plot. `optical_summary` picked its geometry from the display mode, so leaving the chart on RAW published the raw detector contrast as `measured_log_contrast`. A1's amplitude sweep settles on that value against a target `a`: it would never settle, time out at every point, and write a wrong `measured_a` into each sweep sidecar. The geometry is now always the rejected complement; the display mode is presentational. A withheld `a` now reports which gate rejected the window instead of silently showing nothing. Also in the photodiode plugin: - dark level is a measured setting with a capture action, applied to both the detector samples and the I_tot anchor so it cancels out of the complement instead of biasing it; `dark_id` names it honestly - phase-0 marker frames are written into the .pdq, so a recorded run stays phase-attributable offline - `save_cache_snapshot` copies the ring and releases the lock before writing the CSV, instead of blocking the reader across millions of writes - the UI mirror keeps the operator's connect intent rather than clearing it every control tick - a 0-byte read backs off instead of spinning a core - the spectrum max-hold seeds each bucket with its own first bin --- plugins/stage-a-photodiode/src/lib.rs | 603 +++++++++++++++++++++----- 1 file changed, 503 insertions(+), 100 deletions(-) diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 19313c7..1e845eb 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -41,8 +41,8 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; use stage_a_io::{ - estimate_contrast, AdcCalibration, ContrastGeometry, FrameParser, ParseEvent, PdqWriter, - StreamIntegrity, + estimate_contrast, AdcCalibration, ContrastGeometry, EstimateError, FrameParser, ParseEvent, + PdqWriter, StreamIntegrity, }; use stage_a_plugin_contract::{ ClientId, ConnectionStateV1, FreshnessV1, LeaseId, LeaseSnapshotV1, OwnerInstanceId, @@ -598,7 +598,14 @@ fn read_frames( let mut buf = [0_u8; 4_096]; while !stop.load(Ordering::Relaxed) { let read = match port.read(&mut buf) { - Ok(0) => continue, + // A 0-byte read is EOF (e.g. a yanked USB device before the OS + // surfaces an error). Spinning here burns a core while the UI + // still says "reading", so back off and let the timeout path + // report the stall. + Ok(0) => { + std::thread::sleep(Duration::from_millis(5)); + continue; + } Ok(read) => read, Err(err) if err.kind() == std::io::ErrorKind::TimedOut => continue, Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue, @@ -613,40 +620,7 @@ fn read_frames( parser.extend(&buf[..read]); let mut changed = false; while let Some(event) = parser.next_event() { - match event { - ParseEvent::Frame(frame) => { - if let Some(marker) = frame.marker() { - if let Ok(mut state) = shared.lock() { - state.push_marker(marker.sample_index); - } - changed = true; - continue; - } - let Some(codes) = frame.samples() else { - continue; // Control/summary frames are not expected here. - }; - record_frame(recording, &frame, codes.len()); - if let Ok(mut state) = shared.lock() { - state.ingest( - frame.header.first_sample_index, - frame.header.sample_rate_hz, - frame.header.dropped_samples, - &codes, - ); - } - changed = true; - } - ParseEvent::Corruption { - skipped_bytes, - crc_failures, - } => { - if let Ok(mut state) = shared.lock() { - state.resync_bytes += skipped_bytes as u64; - state.crc_failures += crc_failures as u64; - } - changed = true; - } - } + changed |= ingest_parse_event(event, shared, recording); } if changed { generation.fetch_add(1, Ordering::Relaxed); @@ -654,6 +628,54 @@ fn read_frames( } } +/// Applies one parsed stream event to the ring and to any active recording. +/// Split out of [`read_frames`] so the recording/ingest contract is testable +/// without a serial port. Returns whether anything observable changed. +fn ingest_parse_event( + event: ParseEvent, + shared: &Mutex, + recording: &SharedRecording, +) -> bool { + match event { + ParseEvent::Frame(frame) => { + if let Some(marker) = frame.marker() { + // Record before the early return: the phase-0 marker is what + // makes a recorded run phase-attributable offline, so it has + // to reach the .pdq as well as the live ring. It carries no + // samples, hence a sample count of 0. + record_frame(recording, &frame, 0); + if let Ok(mut state) = shared.lock() { + state.push_marker(marker.sample_index); + } + return true; + } + let Some(codes) = frame.samples() else { + return false; // Control/summary frames are not expected here. + }; + record_frame(recording, &frame, codes.len()); + if let Ok(mut state) = shared.lock() { + state.ingest( + frame.header.first_sample_index, + frame.header.sample_rate_hz, + frame.header.dropped_samples, + &codes, + ); + } + true + } + ParseEvent::Corruption { + skipped_bytes, + crc_failures, + } => { + if let Ok(mut state) = shared.lock() { + state.resync_bytes += skipped_bytes as u64; + state.crc_failures += crc_failures as u64; + } + true + } + } +} + pub struct StageAPhotodiodePlugin { enabled: bool, runtime_role: PluginRuntimeRole, @@ -677,6 +699,12 @@ pub struct StageAPhotodiodePlugin { port_hint: String, mode: Mode, reference_volts: f64, + /// Measured dark level in photodiode volts (beam blocked). Applied to both + /// the detector samples and the `reference_volts` anchor, so it cancels out + /// of the rejected-complement contrast rather than biasing it — its job is + /// to keep the two sides consistent and to record the calibration that the + /// reading was taken under. Captured via the "Capture dark" action. + dark_volts: f64, window_s: f64, avg_samples: usize, avg_sync_freq_hz: f64, @@ -688,6 +716,7 @@ pub struct StageAPhotodiodePlugin { press_save_snapshot: PressLatch, press_record_start: PressLatch, press_record_stop: PressLatch, + press_capture_dark: PressLatch, } /// Forwards momentary button presses across the host's UI-mirror → live-worker @@ -771,6 +800,7 @@ impl Default for StageAPhotodiodePlugin { port_hint: "auto".into(), mode: Mode::Raw, reference_volts: 3.3, + dark_volts: 0.0, window_s: 10.0, avg_samples: 4, avg_sync_freq_hz: 0.0, @@ -780,6 +810,7 @@ impl Default for StageAPhotodiodePlugin { press_save_snapshot: PressLatch::default(), press_record_start: PressLatch::default(), press_record_stop: PressLatch::default(), + press_capture_dark: PressLatch::default(), } } } @@ -789,6 +820,44 @@ impl StageAPhotodiodePlugin { self.reader.is_some() } + /// The ADC calibration handed to the contrast estimator, including the + /// measured dark level. + fn adc_calibration(&self) -> AdcCalibration { + AdcCalibration { + volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, + offset_volts: 0.0, + dark_volts: self.dark_volts, + full_scale_code: ADC_MAX_CODE as u16, + } + } + + /// Captures the dark level as the mean of the current ring: the operator + /// blocks the beam, presses the button, and every later contrast is + /// dark-corrected against it. + fn capture_dark(&mut self) -> Result<(), String> { + let mean = { + let state = self + .shared + .lock() + .map_err(|_| "photodiode state lock poisoned".to_owned())?; + if state.samples.is_empty() { + return Err("no samples cached yet — connect and stream first".into()); + } + let sum: u64 = state.samples.iter().map(|&code| u64::from(code)).sum(); + code_to_volts(sum as f64 / state.samples.len() as f64) + }; + if mean >= self.reference_volts { + return Err(format!( + "dark level {mean:.4} V is not below the I_tot reference \ + {:.4} V — is the beam actually blocked?", + self.reference_volts + )); + } + self.dark_volts = mean; + self.last_save_note = Some(format!("dark level captured: {mean:.4} V")); + Ok(()) + } + fn connect(&mut self) { if self.reader.is_some() { return; @@ -1444,42 +1513,64 @@ impl StageAPhotodiodePlugin { } } - /// Live optical log-contrast `a` from the trailing ring window. The ADC - /// always measures the rejected diode `I_pd`, so the display mode selects - /// the geometry: RAW reports the raw detector contrast (`Direct`), - /// EXCITATION reports the excitation contrast (`RejectedComplement`) using - /// `reference_volts` as the total-power anchor `I_tot`. `None` when there is - /// no valid window or, in EXCITATION mode, no valid anchor. + /// Live optical log-contrast `a` from the trailing ring window. + /// + /// The detector sits behind the PBS reject port and measures the rejected + /// complement `I_pd = I_tot - I_exc` — that is a property of the optical + /// bench, settled by construction (knowledge base: + /// `setup/optical-path.md`), not of what the operator chose to plot. So the + /// geometry is always [`ContrastGeometry::RejectedComplement`] anchored on + /// `reference_volts`, and `measured_log_contrast` is always the *excitation* + /// contrast `a = ln(I_exc,max / I_exc,min)`. + /// + /// The display [`Mode`] is presentational only. It must never reach this + /// function: A1's amplitude sweep settles on this value against a target + /// `a`, so letting a display toggle change its meaning would silently + /// retarget the sweep and write a wrong `measured_a` into every sidecar. + /// + /// `None` when there is no valid window or no valid total-power anchor. fn optical_summary(&self, samples: &VecDeque) -> Option { + self.optical_summary_result(samples).ok() + } + + /// [`Self::optical_summary`], keeping the rejection reason so the status + /// readout can explain *why* `a` is being withheld instead of silently + /// showing nothing. + fn optical_summary_result( + &self, + samples: &VecDeque, + ) -> Result { let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); let window: Vec = samples.iter().skip(start).copied().collect(); - let calibration = AdcCalibration { - volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, - offset_volts: 0.0, - dark_volts: 0.0, - full_scale_code: ADC_MAX_CODE as u16, - }; - let geometry = match self.mode { - Mode::Raw => ContrastGeometry::Direct, - Mode::Excitation => ContrastGeometry::RejectedComplement { - total_power_volts: self.reference_volts, - }, + let calibration = self.adc_calibration(); + // `ContrastGeometry::RejectedComplement` wants the *dark-corrected* + // I_tot, and the estimator dark-corrects the detector samples. The + // reference is a reading from the same DC-coupled detector, so it + // carries the same dark offset and has to be corrected the same way. + // Correcting only one side is what would bias `a`; corrected on both, + // the dark term cancels out of the complement exactly (it is a + // difference of two readings), which is the physically right answer. + let geometry = ContrastGeometry::RejectedComplement { + total_power_volts: self.reference_volts - self.dark_volts, }; - let estimate = estimate_contrast(&window, &calibration, geometry).ok()?; + let estimate = estimate_contrast(&window, &calibration, geometry)?; let run_id = self .lease .as_ref() .and_then(|lease| lease.run_id.clone()) .unwrap_or_else(|| RunId::from("live")); - Some(PhotodiodeOpticalSummaryV1 { + Ok(PhotodiodeOpticalSummaryV1 { run_id, calibration: PhotodiodeCalibrationV1 { adc_calibration_id: "adc-default".into(), - dark_id: "dark-0".into(), - anchor_id: match self.mode { - Mode::Raw => "detector-direct".into(), - Mode::Excitation => "reference-volts".into(), + // Name the dark level honestly: consumers must be able to tell + // a measured dark from the un-measured zero default. + dark_id: if self.dark_volts > 0.0 { + "dark-measured".into() + } else { + "dark-none".into() }, + anchor_id: "reference-volts".into(), dark_volts: calibration.dark_volts, total_power_volts: self.reference_volts, }, @@ -1487,6 +1578,10 @@ impl StageAPhotodiodePlugin { log_contrast_stddev: None, excitation_min_volts: estimate.v_min_volts, excitation_max_volts: estimate.v_max_volts, + // Both geometries are dark-referenced (`reference_volts` is the + // dark-corrected `I_tot`), so the excitation minimum *is* the + // margin above dark. Same number as `excitation_min_volts` by + // construction; kept because the contract publishes both. excitation_headroom_volts: estimate.v_min_volts, low_clip_fraction: estimate.low_clip_fraction, high_clip_fraction: estimate.high_clip_fraction, @@ -1496,10 +1591,11 @@ impl StageAPhotodiodePlugin { }) } - /// Locks the ring and returns the current optical log-contrast summary. - fn latest_optical(&self) -> Option { + /// Locks the ring and returns the current optical log-contrast summary, + /// keeping the rejection reason so the caller can explain a withheld `a`. + fn latest_optical_result(&self) -> Option> { let state = self.shared.lock().ok()?; - self.optical_summary(&state.samples) + (!state.samples.is_empty()).then(|| self.optical_summary_result(&state.samples)) } fn control_summary(&self) -> PhotodiodeSummaryV1 { @@ -1640,7 +1736,12 @@ impl StageAPhotodiodePlugin { if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { self.last_error = Some(error); } - self.connect_requested = false; + // Deliberately keep `connect_requested`: it is the operator's + // *intent*, and this branch is what the UI mirror runs on every + // control tick. Clearing it there resets the checkbox before the + // host can sample it, so the live worker never sees the request. + // `connect()` is already guarded on the role, so the intent alone + // is inert here; the worker acts on it below. self.disconnect(); self.lease = None; return; @@ -1662,23 +1763,45 @@ impl StageAPhotodiodePlugin { let dir = self.resolved_data_dir()?; let slug = timestamp_slug(); let csv_path = dir.join(format!("pd_cache_{slug}.csv")); - let state = self - .shared - .lock() - .map_err(|_| "photodiode state lock poisoned".to_owned())?; - if state.samples.is_empty() || state.rate_hz == 0 { - return Err("no samples cached yet".into()); - } + // Copy the ring out under the lock and release it before touching the + // filesystem: holding it across up to RING_MAX_SAMPLES writeln! calls + // blocks the reader thread, overruns the serial input buffer and shows + // up as dropped samples plus a segment restart in any recording that is + // in flight. + let (samples, rate_hz, ring_first_index, cache_seconds, integrity) = { + let state = self + .shared + .lock() + .map_err(|_| "photodiode state lock poisoned".to_owned())?; + if state.samples.is_empty() || state.rate_hz == 0 { + return Err("no samples cached yet".into()); + } + let samples: Vec = state.samples.iter().copied().collect(); + let integrity = json!({ + "resync_bytes": state.resync_bytes, + "crc_failures": state.crc_failures, + "segment_restarts": state.segments, + "device_dropped_samples": state.device_dropped, + }); + ( + samples, + state.rate_hz, + state.ring_first_index, + state.cache_seconds, + integrity, + ) + }; + std::fs::create_dir_all(&dir) .map_err(|err| format!("creating {} failed: {err}", dir.display()))?; let file = File::create(&csv_path) .map_err(|err| format!("creating {} failed: {err}", csv_path.display()))?; let mut writer = BufWriter::new(file); - let rate = f64::from(state.rate_hz); + let rate = f64::from(rate_hz); writeln!(writer, "sample_index,t_s,code,volts") .map_err(|err| format!("writing CSV failed: {err}"))?; - for (offset, &code) in state.samples.iter().enumerate() { - let index = state.ring_first_index + offset as u64; + for (offset, &code) in samples.iter().enumerate() { + let index = ring_first_index + offset as u64; writeln!( writer, "{index},{:.9},{code},{:.6}", @@ -1691,28 +1814,23 @@ impl StageAPhotodiodePlugin { .flush() .map_err(|err| format!("writing CSV failed: {err}"))?; + let sample_count = samples.len(); let sidecar = json!({ "kind": "cache_snapshot", "created_utc": slug, "port": self.port_hint, - "sample_rate_hz": state.rate_hz, - "samples": state.samples.len(), - "first_sample_index": state.ring_first_index, - "cache_seconds": state.cache_seconds, + "sample_rate_hz": rate_hz, + "samples": sample_count, + "first_sample_index": ring_first_index, + "cache_seconds": cache_seconds, "csv_path": csv_path, "adc": { "bits": 12, "full_scale_volts": ADC_FULL_SCALE_VOLTS }, "display_mode": self.mode.name(), "reference_volts": self.reference_volts, + "dark_volts": self.dark_volts, "time_base": "t_s = sample_index / sample_rate_hz, device clock, segment-relative", - "integrity": { - "resync_bytes": state.resync_bytes, - "crc_failures": state.crc_failures, - "segment_restarts": state.segments, - "device_dropped_samples": state.device_dropped, - }, + "integrity": integrity, }); - let sample_count = state.samples.len(); - drop(state); write_json(&csv_path.with_extension("json"), &sidecar)?; self.last_save_note = Some(format!( "saved cache {} ({sample_count} samples)", @@ -2006,7 +2124,10 @@ impl StageAPhotodiodePlugin { y: peak, }); peak = 0.0; - peak_freq = freq; + // Seed the *next* bucket with its own first bin. Seeding with + // `freq` (the bin that just closed this bucket) put a flat + // bucket's point one bucket to the left. + peak_freq = (k + 1) as f64 * rate / n as f64; in_bucket = 0; } } @@ -2427,7 +2548,8 @@ impl Plugin for StageAPhotodiodePlugin { if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { self.last_error = Some(error); } - self.connect_requested = false; + // Demoting to the UI mirror drops the hardware, not the operator's + // connect intent — see `apply_execution_context`. self.disconnect(); self.lease = None; self.effects_allowed = false; @@ -2452,7 +2574,8 @@ impl Plugin for StageAPhotodiodePlugin { if let Err(error) = self.finalize_recording(PdqTerminationV1::Aborted) { self.last_error = Some(error); } - self.connect_requested = false; + // Runs every replayed frame, so it must not clear the intent + // either — the port stays closed because `connect()` is guarded. self.disconnect(); self.lease = None; } @@ -2634,6 +2757,32 @@ impl Plugin for StageAPhotodiodePlugin { default: self.reference_volts, }, }, + SettingItem { + key: "dark_volts".into(), + label: "Dark level".into(), + tooltip: Some( + "Measured dark level in photodiode volts (beam blocked). The \ + detector is DC-coupled, so the published contrast a is biased low \ + while this is 0." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.0, + max: ADC_FULL_SCALE_VOLTS, + speed: 0.001, + default: self.dark_volts, + }, + }, + SettingItem { + key: "capture_dark".into(), + label: "Capture dark".into(), + tooltip: Some( + "Block the beam, then press: takes the mean of the current cache \ + as the dark level." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, SettingItem { key: "window_s".into(), label: "Chart window".into(), @@ -2840,6 +2989,8 @@ impl Plugin for StageAPhotodiodePlugin { // live worker (see PressLatch). "record_start" => Some(self.press_record_start.value()), "record_stop" => Some(self.press_record_stop.value()), + "dark_volts" => Some(json!(self.dark_volts)), + "capture_dark" => Some(self.press_capture_dark.value()), "save_snapshot" => Some(self.press_save_snapshot.value()), _ => None, } @@ -2961,6 +3112,23 @@ impl Plugin for StageAPhotodiodePlugin { } Ok(()) } + "dark_volts" => { + let volts = value.as_f64().ok_or("dark_volts must be a number")?; + self.dark_volts = volts.clamp(0.0, ADC_FULL_SCALE_VOLTS); + Ok(()) + } + "capture_dark" => { + // Edge-guarded like every other effectful arm: the host + // re-applies the whole settings snapshot on each sync. + if self.press_capture_dark.accept(&value) { + match self.capture_dark() { + Ok(()) => self.last_error = None, + Err(err) => self.last_error = Some(err), + } + self.generation.fetch_add(1, Ordering::Relaxed); + } + Ok(()) + } "save_snapshot" => { // Edge-guarded: the host re-applies the full settings snapshot // on every sync, and an unguarded arm wrote one cache file per @@ -3021,17 +3189,28 @@ impl Plugin for StageAPhotodiodePlugin { ))); } } - if let Some(optical) = self.latest_optical() { - let label = match self.mode { - Mode::Raw => "a_raw (detector)", - Mode::Excitation => "a (excitation)", - }; - entries.push(StatusEntry::Text(format!( - "{label} = {:.3} (I {:.4}..{:.4} V)", - optical.measured_log_contrast, - optical.excitation_min_volts, - optical.excitation_max_volts - ))); + match self.latest_optical_result() { + // Always the excitation contrast: the geometry follows the bench, + // not the display mode. + Some(Ok(optical)) => { + entries.push(StatusEntry::Text(format!( + "a (excitation) = {:.3} (I {:.4}..{:.4} V)", + optical.measured_log_contrast, + optical.excitation_min_volts, + optical.excitation_max_volts + ))); + if self.dark_volts <= 0.0 { + entries.push(StatusEntry::Text( + "a is uncorrected for dark — capture a dark level".into(), + )); + } + } + // A withheld `a` is a fail-closed refusal, not an absence of data: + // say which gate rejected the window so the operator can fix it. + Some(Err(error)) => { + entries.push(StatusEntry::Text(format!("a unavailable: {error}"))); + } + None => {} } if let Ok(state) = self.shared.lock() { if let Some(period_samples) = state.marker_period_samples() { @@ -3166,6 +3345,141 @@ mod tests { plugin } + /// A clean rejected-port sine: the detector swings around `center` while + /// the excitation is its complement against `I_tot`. + fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> VecDeque { + (0..count) + .map(|i| { + let phase = 2.0 * std::f64::consts::PI * (i as f64) * 8.0 / count as f64; + (center + amplitude * phase.sin()).round().clamp(0.0, 4_095.0) as u16 + }) + .collect() + } + + #[test] + fn published_contrast_is_the_excitation_contrast_in_both_display_modes() { + // The detector sits behind the PBS reject port whatever the operator + // is plotting, so a display toggle must not move a published + // scientific quantity. A1's amplitude sweep settles on this value. + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + + plugin.mode = Mode::Raw; + let raw = plugin.optical_summary(&samples).expect("raw display"); + plugin.mode = Mode::Excitation; + let excitation = plugin + .optical_summary(&samples) + .expect("excitation display"); + + assert_eq!(raw.measured_log_contrast, excitation.measured_log_contrast); + assert_eq!(raw.calibration.anchor_id, "reference-volts"); + assert_eq!(excitation.calibration.anchor_id, "reference-volts"); + + // And it really is the complement contrast, not ln(v_max/v_min) of the + // detector trace. + let detector_direct = ((1_600.0_f64 + 700.0) / (1_600.0 - 700.0)).ln(); + assert!( + (raw.measured_log_contrast - detector_direct).abs() > 0.1, + "published a={} collapsed to the detector-direct contrast", + raw.measured_log_contrast + ); + } + + #[test] + fn captured_dark_level_reaches_the_estimator_and_is_named() { + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + + let undarkened = plugin.optical_summary(&samples).expect("no dark yet"); + assert_eq!(undarkened.calibration.dark_id, "dark-none"); + assert_eq!(undarkened.calibration.dark_volts, 0.0); + + plugin.dark_volts = 0.05; + let darkened = plugin.optical_summary(&samples).expect("with dark"); + assert_eq!(darkened.calibration.dark_id, "dark-measured"); + assert_eq!(darkened.calibration.dark_volts, 0.05); + // A DC dark offset is common to the detector samples and to the + // reference reading, so it cancels out of the complement. Anything + // else means one of the two sides is being corrected without the + // other — which is what would actually bias `a`. + assert!( + (darkened.measured_log_contrast - undarkened.measured_log_contrast).abs() < 1e-9, + "dark did not cancel: {} vs {}", + darkened.measured_log_contrast, + undarkened.measured_log_contrast + ); + } + + #[test] + fn a_dark_offset_on_only_one_side_would_bias_the_contrast() { + // Guards the invariance above against a regression that dark-corrects + // the detector but leaves the anchor raw (or vice versa): that is the + // asymmetry the estimator contract warns about. + let calibration = AdcCalibration { + volts_per_code: ADC_FULL_SCALE_VOLTS / ADC_MAX_CODE, + offset_volts: 0.0, + dark_volts: 0.05, + full_scale_code: ADC_MAX_CODE as u16, + }; + let samples: Vec = rejected_port_samples(1_600.0, 700.0, 4_096) + .into_iter() + .collect(); + let consistent = estimate_contrast( + &samples, + &calibration, + ContrastGeometry::RejectedComplement { + total_power_volts: 3.0 - 0.05, + }, + ) + .expect("consistent"); + let asymmetric = estimate_contrast( + &samples, + &calibration, + ContrastGeometry::RejectedComplement { + total_power_volts: 3.0, + }, + ) + .expect("anchor left raw"); + assert!( + (consistent.a - asymmetric.a).abs() > 1e-3, + "the asymmetry must be observable, else this test proves nothing" + ); + } + + #[test] + fn capture_dark_refuses_a_level_at_or_above_the_anchor() { + let mut plugin = live_plugin(); + plugin.reference_volts = 0.5; + if let Ok(mut state) = plugin.shared.lock() { + state.ingest(0, 20_000, 0, &[4_000; 256]); + } + let err = plugin.capture_dark().expect_err("beam clearly not blocked"); + assert!(err.contains("is not below the I_tot reference"), "{err}"); + assert_eq!(plugin.dark_volts, 0.0); + } + + #[test] + fn the_ui_mirror_keeps_the_operators_connect_intent() { + // The mirror runs `apply_execution_context` every control tick. If it + // clears the intent, the host samples `connect` as false and the live + // worker never opens the port. + let mut plugin = StageAPhotodiodePlugin::default(); + plugin.set_runtime_role(PluginRuntimeRole::UiMirror); + plugin.set_setting("connect", json!(true)).expect("connect"); + assert!(plugin.connect_requested); + + plugin.apply_execution_context(&live_execution()); + assert!( + plugin.connect_requested, + "the mirror cleared the connect intent" + ); + assert_eq!(plugin.get_setting("connect"), Some(json!(true))); + // ...but it must not have actually opened anything. + assert!(!plugin.connected()); + } + fn service_request( plugin: &StageAPhotodiodePlugin, id: u64, @@ -3576,6 +3890,95 @@ mod tests { dir } + fn marker_frame(sequence: u32, sample_index: u64) -> Frame { + let mut payload = Vec::with_capacity(16); + payload.extend_from_slice(&sample_index.to_le_bytes()); + payload.extend_from_slice(&0_u32.to_le_bytes()); // tick_us + payload.push(1); // level + payload.push(0); // source + payload.extend_from_slice(&[0, 0]); // reserved + Frame::build( + FrameHeader { + version: stage_a_io::wire::PROTOCOL_VERSION, + frame_type: FrameType::Marker, + flags: 0, + sequence, + payload_bytes: 0, + first_sample_index: sample_index, + sample_rate_hz: MOCK_RATE_HZ, + dropped_samples: 0, + crc32: 0, + }, + payload, + ) + } + + #[test] + fn phase_zero_markers_are_written_into_the_recording() { + // Without the marker frames a recorded run cannot be phase-attributed + // offline, which is the whole point of the .pdq evidence file. + let dir = temp_dir("marker-record"); + let pdq_path = dir.join("run.pdq"); + let shared = Arc::new(Mutex::new(SharedState::default())); + let recording: SharedRecording = Arc::new(Mutex::new(Some(RecordingSink { + writer: PdqWriter::create(&pdq_path).expect("create pdq"), + pdq_path: pdq_path.clone(), + sidecar_path: dir.join("run.json"), + pdq_path_label: "run.pdq".into(), + sidecar_path_label: "run.json".into(), + run_id: RunId::from("test"), + opened_at_unix_ms: 0, + stream_epoch: 0, + first_sample_index: None, + metadata: BTreeMap::new(), + started_slug: "slug".into(), + samples_written: 0, + write_error: None, + start_crc_failures: 0, + start_resync_bytes: 0, + start_device_dropped: 0, + start_segments: 0, + }))); + + let codes = [100_u16, 200, 300, 400]; + assert!(ingest_parse_event( + ParseEvent::Frame(mock_sample_frame(0, 0, &codes)), + &shared, + &recording, + )); + assert!(ingest_parse_event( + ParseEvent::Frame(marker_frame(1, 2)), + &shared, + &recording, + )); + + // The marker still reaches the live ring... + assert_eq!( + shared.lock().unwrap().markers.iter().copied().last(), + Some(2) + ); + // ...and the sample count is unaffected by the marker frame. + let sink = recording.lock().unwrap().take().expect("sink"); + assert_eq!(sink.samples_written, codes.len() as u64); + sink.writer + .finish(StreamIntegrity::default()) + .expect("finish pdq"); + + let mut reader = stage_a_io::PdqReader::open(&pdq_path).expect("open pdq"); + let mut frame_types = Vec::new(); + while let Some(event) = reader.next_event().expect("read event") { + if let stage_a_io::PdqReadEvent::Frame(frame) = event { + frame_types.push(frame.header.frame_type); + } + } + assert!( + frame_types.contains(&FrameType::Marker), + "the .pdq holds no marker frame: {frame_types:?}" + ); + assert!(frame_types.contains(&FrameType::SamplesU16)); + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn cache_snapshot_writes_csv_and_sidecar() { let dir = temp_dir("snapshot"); From 029638a4f97878de8d34b52af937aafb7d432004 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:06:08 +0200 Subject: [PATCH 23/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20stop=20set?= =?UTF-8?q?tings=20syncs=20from=20overwriting=20a=20running=20protocol=20s?= =?UTF-8?q?tep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `send_modulation` was silenced for an automation lease and a calibration sweep but not for a running protocol, which queues its steps into the same `pending` slot. Because the host re-applies the whole settings snapshot on every sync, any settings change — from this plugin or another — dropped the operator's armed drive on top of the protocol's queued step, and the board held it until the next step boundary. Also in the modulation plugin: - `mod_freq_mhz` is parsed once as f64 and rounded to millihertz; the second u64 parse returned None as soon as the firmware echoed a decimal, which published frequency_millihz: 0 and cost A1 its fallback modulation period - a leased `SetOpticalDepth` now parks the operator's armed depth and `end_lease` restores it, so the board no longer holds the last sweep point's depth after an A1 amplitude sweep finishes --- plugins/stage-a-modulation/src/lib.rs | 186 +++++++++++++++++++++++--- 1 file changed, 169 insertions(+), 17 deletions(-) diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index a77c6f2..2dc3e51 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -561,10 +561,16 @@ fn apply_reply_fields(state: &mut DeviceState, fields: &BTreeMap if let Some(wave) = fields.get("mod_wave") { let level = fields.get("mod_level").map(String::as_str).unwrap_or("?"); let min = fields.get("mod_min").map(String::as_str).unwrap_or("?"); - let freq_mhz = fields + // Parse the echoed frequency exactly once, as f64. Parsing it a second + // time as u64 silently yielded None the moment the firmware echoed a + // decimal ("10000.0"): `board_echo_target` then published + // frequency_millihz: 0, A1 rejected it, and A1 lost its only fallback + // modulation period whenever the EXT_TRIGGER markers were absent. + let freq_millihz = fields .get("mod_freq_mhz") .and_then(|v| v.parse::().ok()) - .unwrap_or(0.0); + .filter(|hz| hz.is_finite() && *hz >= 0.0); + let freq_mhz = freq_millihz.unwrap_or(0.0); state.board_mod = if wave == "SINE" || wave == "SQUARE" { format!("{wave} {min}..{level} @ {:.3} Hz", freq_mhz / 1_000.0) } else { @@ -573,7 +579,7 @@ fn apply_reply_fields(state: &mut DeviceState, fields: &BTreeMap state.board_wave = Some(wave.clone()); state.board_level = fields.get("mod_level").and_then(|v| v.parse().ok()); state.board_min = fields.get("mod_min").and_then(|v| v.parse().ok()); - state.board_freq_millihz = fields.get("mod_freq_mhz").and_then(|v| v.parse().ok()); + state.board_freq_millihz = freq_millihz.map(|hz| hz.round() as u64); } } @@ -900,6 +906,9 @@ pub struct StageAModulationPlugin { // -- optical drive inversion (OPTICAL_* modes) -- /// Requested optical log-modulation depth `a = ln(I_max / I_min)`. depth_a: f64, + /// The operator's armed `depth_a`, parked while a lease drives the optical + /// depth (A1's amplitude sweep) and restored by [`Self::end_lease`]. + armed_depth_a: Option, /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. operating_point: f64, @@ -973,6 +982,7 @@ impl Default for StageAModulationPlugin { mode: Mode::Const, frequency_hz: 10.0, depth_a: 0.5, + armed_depth_a: None, operating_point: 0.5, v_null_dac: 0, v_pi_dac: 2_048, @@ -1240,14 +1250,20 @@ impl StageAModulationPlugin { /// Queues one MOD command carrying the complete current drive settings; /// newer changes overwrite queued ones (drag coalescing). /// - /// Silent while another owner holds the DAC. Besides an automation lease - /// that now includes a calibration sweep: the host re-applies the *whole* - /// settings snapshot on every sync, and most handlers here call this - /// unconditionally, so without the guard every sync would re-arm the - /// operator's drive on top of the code the sweep just commanded — the - /// sweep would measure the armed waveform instead of its own staircase. + /// Silent while another owner holds the DAC: an automation lease, a + /// calibration sweep, or a running protocol. The host re-applies the + /// *whole* settings snapshot on every sync, and most handlers here call + /// this unconditionally, so without the guard every sync would re-arm the + /// operator's drive on top of the code the current owner just commanded — + /// the sweep would measure the armed waveform instead of its own + /// staircase, and a protocol step would be overwritten mid-step and held + /// until the next step boundary. fn send_modulation(&mut self) { - if self.link.is_none() || self.lease.is_some() || self.sweep.is_some() { + if self.link.is_none() + || self.lease.is_some() + || self.sweep.is_some() + || self.protocol_active() + { return; } let command = match self.drive_command() { @@ -1887,7 +1903,7 @@ impl StageAModulationPlugin { self.deferred_release_ack_published = false; return Ok(response); } - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.shared .fail_closed_on_stop @@ -1976,6 +1992,10 @@ impl StageAModulationPlugin { )); } }; + // Remember what the operator had armed before the first + // sweep point, so `end_lease` can hand it back. Only the + // first one: later points must not overwrite the original. + self.armed_depth_a.get_or_insert(previous); *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { commands: vec![command], purpose: "MOD", @@ -2131,6 +2151,23 @@ impl StageAModulationPlugin { } } + /// Ends the current lease and gives the operator their armed drive back. + /// + /// A leased `SetOpticalDepth` (A1's amplitude sweep) writes straight into + /// `depth_a`. Without this the modulation UI kept showing — and the board + /// kept holding — the last sweep point's depth after the sweep finished, + /// rather than what the operator had armed. The calibration sweep already + /// restores through `Sweep::restore`; this is the leased equivalent. + fn end_lease(&mut self) { + self.lease = None; + if let Some(depth) = self.armed_depth_a.take() { + self.depth_a = depth; + // Re-arm the board only if nobody else now owns the DAC; + // `send_modulation` is itself guarded. + self.send_modulation(); + } + } + fn expire_lease_if_needed(&mut self) { let expired = self .lease @@ -2152,7 +2189,7 @@ impl StageAModulationPlugin { purpose: "LEASE_EXPIRED_SAFE_OFF", meta: None, }); - self.lease = None; + self.end_lease(); self.last_error = Some("automation lease expired; queued STOP + output off".into()); self.shared.bump(); } @@ -2175,7 +2212,7 @@ impl StageAModulationPlugin { return; } if self.deferred_release_ack_published { - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.deferred_release_ack_published = false; self.shared @@ -2201,7 +2238,7 @@ impl StageAModulationPlugin { .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); } - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.deferred_release_ack_published = false; return; @@ -2647,7 +2684,7 @@ impl Plugin for StageAModulationPlugin { .fail_closed_on_stop .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); - self.lease = None; + self.end_lease(); self.deferred_release_request = None; } } @@ -2662,7 +2699,7 @@ impl Plugin for StageAModulationPlugin { .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); } - self.lease = None; + self.end_lease(); self.deferred_release_request = None; self.deferred_release_ack_published = false; } @@ -2686,7 +2723,7 @@ impl Plugin for StageAModulationPlugin { .fail_closed_on_stop .store(self.lease.is_some(), Ordering::Relaxed); self.disconnect(); - self.lease = None; + self.end_lease(); self.last_error = Some("disconnected: replay mode".into()); } } @@ -3337,6 +3374,13 @@ impl Plugin for StageAModulationPlugin { } self.send_modulation(); } + // An edit made while a lease drives the depth is withheld from + // the board (`send_modulation` is guarded), so it has to land + // in the parked value or it would be lost when the lease ends + // — same rule the calibration sweep follows. + if self.armed_depth_a.is_some() { + self.armed_depth_a = Some(self.depth_a); + } Ok(()) } "operating_point" => { @@ -4718,6 +4762,114 @@ level = 750 plugin.disconnect(); } + #[test] + fn ending_a_lease_restores_the_operators_armed_optical_depth() { + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + plugin.method = DriveMethod::Calibrated; + plugin.mode = Mode::Sine; + plugin.depth_a = 0.4; // what the operator armed + + let acquire = service_request( + &plugin, + 60, + "stage-a-a1", + ModulationCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + plugin.handle_service_request(&acquire, &live_execution()); + + // Two sweep points: only the first must be remembered as "armed". + for (id, milli) in [(61_u64, 900_u32), (62, 1_250)] { + let point = service_request( + &plugin, + id, + "stage-a-a1", + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: milli, + }, + None, + ); + let reply = plugin.handle_service_request(&point, &live_execution()); + assert!( + matches!(reply.outcome, PluginServiceOutcome::Accepted { .. }), + "sweep point {milli} rejected: {:?}", + reply.outcome + ); + } + assert!((plugin.depth_a - 1.25).abs() < 1e-9, "sweep drives the depth"); + + plugin.end_lease(); + assert!( + (plugin.depth_a - 0.4).abs() < 1e-9, + "armed depth not restored: {}", + plugin.depth_a + ); + assert!(plugin.armed_depth_a.is_none()); + plugin.disconnect(); + } + + #[test] + fn a_running_protocol_owns_the_pending_slot() { + // The host re-applies the whole settings snapshot on every sync. An + // unguarded `send_modulation` would drop the operator's armed drive + // into the slot the protocol step is queued in, and the board would + // hold it until the next step boundary. + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + plugin.connect_requested = true; + plugin.connect(); + wait_until(&plugin, Duration::from_secs(2), |owner| { + owner.device_connected() + }); + + let progress = Arc::new(Mutex::new(ProtocolProgress::default())); + plugin.protocol = Some(ProtocolRun { + progress: Arc::clone(&progress), + stop: Arc::new(AtomicBool::new(false)), + join: None, + }); + assert!(plugin.protocol_active()); + + *plugin.shared.pending.lock().unwrap() = None; + plugin.set_setting("max_level", json!(3_000)).expect("set"); + assert!( + plugin.shared.pending.lock().unwrap().is_none(), + "a settings sync overwrote the protocol's pending slot" + ); + + // Once the protocol finishes, the operator's drive gets through again. + progress.lock().unwrap().finished = true; + assert!(!plugin.protocol_active()); + plugin.set_setting("max_level", json!(3_100)).expect("set"); + assert!(plugin.shared.pending.lock().unwrap().is_some()); + plugin.disconnect(); + } + + #[test] + fn a_decimal_frequency_echo_still_yields_millihertz() { + // Firmware echoing "10000.0" used to parse as u64 -> None, which + // published frequency_millihz: 0 and cost A1 its fallback period. + let mut state = DeviceState::default(); + let mut fields = BTreeMap::new(); + fields.insert("mod_wave".to_owned(), "SINE".to_owned()); + fields.insert("mod_level".to_owned(), "2000".to_owned()); + fields.insert("mod_min".to_owned(), "100".to_owned()); + fields.insert("mod_freq_mhz".to_owned(), "10000.0".to_owned()); + apply_reply_fields(&mut state, &fields); + assert_eq!(state.board_freq_millihz, Some(10_000)); + + // The integer form keeps working. + fields.insert("mod_freq_mhz".to_owned(), "7500".to_owned()); + apply_reply_fields(&mut state, &fields); + assert_eq!(state.board_freq_millihz, Some(7_500)); + } + #[test] fn set_optical_depth_requires_lease_and_a_calibrated_drive() { let mut plugin = live_plugin(); From c1303d77901b376b73ee8d428c15bd8dd38e9249 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:10:35 +0200 Subject: [PATCH 24/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20normalise?= =?UTF-8?q?=20the=20A1=20rolling=20response=20over=20the=20ROI,=20and=20me?= =?UTF-8?q?moise=20the=20fold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `S_p(t)` divided by the whole sensor (`width * height`) while `q_p` on the same screen divided by ROI area minus masked pixels, and the rolling numerator counted events from outside the ROI and from masked pixels. With a small ROI that under-reported `S_p` by the ROI/frame ratio, and the status readout printed both numbers under the same "valid pixels" label. The fold is now built from ROI-filtered events and both quantities divide by `valid_pixel_count()`. `current_fold()` is memoised on a fingerprint of its inputs. It is called from `rolling_dataset`, `latest_rolling` (twice), `current_windows` and `current_response`, each allocating a `Vec` over up to MAX_EVENTS — a single repaint could allocate and discard hundreds of megabytes at bench event rates. Also in the A1 plugin: - the no-EventStore fallback trims by the analysis window instead of growing to MAX_EVENTS and then freezing on a stale buffer while the plots still looked live - a failed pilot-window freeze clears the previously loaded windows, so the sidecar cannot record an earlier pilot's windows as this run's --- plugins/stage-a-a1/src/runtime.rs | 242 ++++++++++++++++++++++++++++-- 1 file changed, 228 insertions(+), 14 deletions(-) diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index b1baedd..64ca65b 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -24,6 +24,7 @@ //! `q_p(a, f)` fit is computed offline from the recordings; the live plot is a //! quicklook. +use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -299,9 +300,10 @@ pub struct StageAA1Plugin { /// anchor the fold to the drive on the camera clock; empty falls back to the /// free-running fold on `T`. camera_markers_us: Vec, - valid_pixels: usize, frame_width: u16, frame_height: u16, + /// Memoised [`StageAA1Plugin::current_fold`], keyed on its inputs. + fold_cache: RefCell)>>, // -- host camera ROI/mask, mirrored from CTX_GLOBAL_SETTINGS -- host_roi: Option, masked_pixels: HashSet<(u16, u16)>, @@ -367,7 +369,7 @@ impl Default for StageAA1Plugin { event_scratch: Vec::new(), analysis_window_ms: DEFAULT_ANALYSIS_WINDOW_MS, camera_markers_us: Vec::new(), - valid_pixels: 0, + fold_cache: RefCell::new(None), frame_width: 0, frame_height: 0, host_roi: None, @@ -466,6 +468,22 @@ impl Recording { } } +/// Fingerprint of everything the phase fold is computed from. Cheap to build +/// (no scan of the event buffer) and exact enough that a stale fold cannot +/// survive a change to any input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct FoldKey { + period_us_bits: u64, + event_count: usize, + first_event_us: Option, + last_event_us: Option, + marker_count: usize, + first_marker_us: Option, + last_marker_us: Option, + roi: Option, + masked_count: usize, +} + impl StageAA1Plugin { fn bump(&mut self) { self.dataset_generation = self.dataset_generation.wrapping_add(1); @@ -523,12 +541,59 @@ impl StageAA1Plugin { } } + /// The current phase fold, memoised. + /// + /// Called several times per repaint (`rolling_dataset`, `latest_rolling` + /// from both `status_dataset` and `status_entries`, `current_windows`, + /// `current_response`). Each fold allocates a `Vec` over up to + /// `MAX_EVENTS` events, so refolding per call threw away hundreds of + /// megabytes per repaint at bench event rates. The cache is keyed on a + /// cheap fingerprint of everything the fold reads, so it invalidates + /// exactly when the inputs move rather than on every `bump()`. fn current_fold(&self) -> Option { + let key = self.fold_key()?; + if let Ok(cache) = self.fold_cache.try_borrow() { + if let Some((cached_key, fold)) = cache.as_ref() { + if *cached_key == key { + return fold.clone(); + } + } + } + let fold = self.compute_fold(); + if let Ok(mut cache) = self.fold_cache.try_borrow_mut() { + *cache = Some((key, fold.clone())); + } + fold + } + + /// Fingerprint of every input [`Self::compute_fold`] reads. `None` when + /// there is no period, i.e. no fold to compute. + fn fold_key(&self) -> Option { + let period_us = self.period_us()?; + Some(FoldKey { + period_us_bits: period_us.to_bits(), + event_count: self.camera_events.len(), + first_event_us: self.camera_events.first().map(|event| event.timestamp_us), + last_event_us: self.camera_events.last().map(|event| event.timestamp_us), + marker_count: self.camera_markers_us.len(), + first_marker_us: self.camera_markers_us.first().copied(), + last_marker_us: self.camera_markers_us.last().copied(), + roi: self.roi(), + masked_count: self.masked_pixels.len(), + }) + } + + fn compute_fold(&self) -> Option { let period_us = self.period_us()?; + // Fold only the events the analysis is normalised over. `q_p` already + // restricts to ROI minus masked pixels; the rolling response divides by + // the same count, so its numerator has to be restricted too or it + // counts events from outside the ROI against an ROI-sized denominator. + let events = self.roi_filtered_events(); let marker_fold = self.is_marker_anchored().then(|| { let expected_hz = 1_000_000.0 / period_us; fold_events( - &self.camera_events, + &events, &self.camera_markers_us, MarkerValidationConfig { expected_frequency_hz: expected_hz, @@ -545,7 +610,27 @@ impl StageAA1Plugin { // blank the live plots — fall back to the free-running fold on T. marker_fold .flatten() - .or_else(|| fold_events_free_running(&self.camera_events, period_us)) + .or_else(|| fold_events_free_running(&events, period_us)) + } + + /// The analysis-window events restricted to the ROI, masked pixels removed. + /// Without an ROI the whole frame is the ROI, so this is a clone. + fn roi_filtered_events(&self) -> Vec { + let Some(roi) = self.roi() else { + return self.camera_events.clone(); + }; + if roi.area() == usize::from(self.frame_width) * usize::from(self.frame_height) + && self.masked_pixels.is_empty() + { + return self.camera_events.clone(); + } + self.camera_events + .iter() + .filter(|event| { + roi.contains(event.x, event.y) && !self.masked_pixels.contains(&(event.x, event.y)) + }) + .copied() + .collect() } /// Optical modulation depth `a` published by the photodiode plugin. @@ -622,6 +707,10 @@ impl StageAA1Plugin { self.note("Pilot windows frozen from the live signal"); } None => { + // Drop whatever was loaded for this measurement: leaving it in + // place let `write_sidecar` record windows from an *earlier* + // pilot as if they had just been frozen from this run. + self.pilot_windows = None; self.note("No live signal to freeze windows — enable Live analysis first"); } } @@ -716,8 +805,13 @@ impl StageAA1Plugin { let sample_times: Vec = (0..samples) .map(|index| first + (last - first) * index / (samples - 1)) .collect(); + // Same denominator as `q_p` (ROI minus masked), against the ROI-filtered + // fold — the two are shown side by side and must mean the same thing. + let Some(valid_pixels) = self.valid_pixel_count() else { + return empty(); + }; let line = |polarity: Polarity| { - rolling_half_period_response(&fold, polarity, self.valid_pixels, &sample_times, None) + rolling_half_period_response(&fold, polarity, valid_pixels, &sample_times, None) .map(|points| points_for(&points, first)) .unwrap_or_default() }; @@ -741,8 +835,9 @@ impl StageAA1Plugin { fn latest_rolling(&self) -> Option<(f64, f64)> { let fold = self.current_fold()?; let at = [fold.validation.last_marker_us]; + let valid_pixels = self.valid_pixel_count()?; let value = |polarity| { - rolling_half_period_response(&fold, polarity, self.valid_pixels, &at, None) + rolling_half_period_response(&fold, polarity, valid_pixels, &at, None) .ok() .and_then(|points| points.first().map(|point| point.run_per_pixel)) }; @@ -2123,7 +2218,6 @@ impl Plugin for StageAA1Plugin { self.camera_events.clear(); self.event_scratch.clear(); self.camera_markers_us.clear(); - self.valid_pixels = 0; self.response_points.clear(); self.pilot_windows = None; self.background_floor = None; @@ -2179,7 +2273,6 @@ impl Plugin for StageAA1Plugin { if !self.live { return; } - self.valid_pixels = usize::from(frame.width()) * usize::from(frame.height()); // Markers (phase-0 sync) only exist on the preview frame, so accumulate // the rising EXT_TRIGGER edges here regardless of the event source. @@ -2216,11 +2309,30 @@ impl Plugin for StageAA1Plugin { // Keep the marker set on the same window as the events. self.camera_markers_us .retain(|&marker| marker >= window_start); - } else if self.camera_events.len() < MAX_EVENTS { + } else { // Fallback (no retained history available): accumulate the - // best-effort preview-frame events. + // best-effort preview-frame events, then trim to the same analysis + // window the exact path uses. Without the trim the buffer grew to + // MAX_EVENTS and then stopped accepting anything at all, so the + // fold silently spanned an ever-widening window and finally froze + // on a stale 4M-event buffer while the plots still looked live. self.camera_events .extend(frame.events().iter().map(ffi_to_camera_event)); + let window_start = window_end.saturating_sub(window_us); + let keep_from = self + .camera_events + .partition_point(|event| event.timestamp_us < window_start); + if keep_from > 0 { + self.camera_events.drain(..keep_from); + } + // Hard ceiling as well: a window longer than the event buffer can + // hold must drop the oldest events, not stop taking new ones. + if self.camera_events.len() > MAX_EVENTS { + let excess = self.camera_events.len() - MAX_EVENTS; + self.camera_events.drain(..excess); + } + self.camera_markers_us + .retain(|&marker| marker >= window_start); } self.bump(); } @@ -2673,7 +2785,6 @@ impl Plugin for StageAA1Plugin { self.camera_events.clear(); self.event_scratch.clear(); self.camera_markers_us.clear(); - self.valid_pixels = 0; } } "window_floor" => { @@ -2759,7 +2870,7 @@ impl Plugin for StageAA1Plugin { entries.push(StatusEntry::Text(format!( "{} events, {} valid pixels; {anchor}", self.camera_events.len(), - self.valid_pixels + self.valid_pixel_count().unwrap_or(0) ))); entries.push(StatusEntry::Text(match self.measured_a() { Some(a) => format!("a = {a:.3} (photodiode)"), @@ -2988,7 +3099,9 @@ mod tests { /// A plugin whose period comes from marker spacing (no fallback frequency). fn plugin_with_markers() -> StageAA1Plugin { StageAA1Plugin { - valid_pixels: 10, + // 10 x 1 sensor, no host ROI => valid_pixel_count() == 10. + frame_width: 10, + frame_height: 1, camera_markers_us: vec![0, 1_000, 2_000, 3_000], ..StageAA1Plugin::default() } @@ -3077,6 +3190,106 @@ mod tests { assert!(plugin.record_response_point().is_err()); } + #[test] + fn the_rolling_response_is_normalised_over_the_roi_not_the_sensor() { + // `q_p` counts ROI-minus-masked pixels; the rolling half-period rate is + // plotted next to it and must agree. Normalising by the whole sensor + // under-reported S_p by the ROI/frame ratio *and* counted events from + // outside the ROI. + let mut plugin = StageAA1Plugin { + frame_width: 10, + frame_height: 10, + camera_markers_us: vec![0, 1_000, 2_000, 3_000], + ..StageAA1Plugin::default() + }; + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 2, + height: 2, + }); + let event = |x: u16, y: u16, timestamp_us: u64| CameraEvent { + timestamp_us, + x, + y, + polarity: Polarity::On, + }; + // Two ON events inside the 2x2 ROI, five well outside it, all inside + // the trailing half period the status readout samples. + plugin.camera_events.push(event(0, 0, 2_800)); + plugin.camera_events.push(event(1, 1, 2_850)); + for x in 5..10_u16 { + plugin.camera_events.push(event(x, 9, 2_900)); + } + + let (on_rate, _) = plugin.latest_rolling().expect("rolling value"); + assert!( + (on_rate - 0.5).abs() < 1e-9, + "expected 2 ROI events over 4 valid pixels, got {on_rate}" + ); + } + + #[test] + fn the_fold_cache_tracks_its_inputs() { + let mut plugin = plugin_with_markers(); + for cycle in 0..8 { + plugin.camera_events.push(on(cycle * 1_000 + 200)); + } + let first = plugin.current_fold().expect("fold"); + // Repeated calls within a repaint must be identical, not merely equal + // to a fresh recomputation. + assert_eq!(plugin.current_fold().as_ref(), Some(&first)); + assert_eq!(plugin.compute_fold().as_ref(), Some(&first)); + + // ...and adding an event inside the marker span must invalidate it. + plugin.camera_events.push(on(2_500)); + plugin.camera_events.sort_by_key(|event| event.timestamp_us); + let second = plugin.current_fold().expect("fold"); + assert_eq!(second.events.len(), first.events.len() + 1); + + // A changed ROI also invalidates, even at identical event counts. + plugin.host_roi = Some(RoiV1 { + x: 0, + y: 0, + width: 1, + height: 1, + }); + let third = plugin.current_fold().expect("fold"); + assert_eq!(third.events.len(), second.events.len()); + plugin.host_roi = Some(RoiV1 { + x: 5, + y: 0, + width: 1, + height: 1, + }); + let fourth = plugin.current_fold().expect("fold"); + assert!( + fourth.events.is_empty(), + "ROI moved off the events but the cache served a stale fold" + ); + } + + #[test] + fn a_failed_pilot_freeze_clears_stale_windows() { + // `scan_measurement_folder` may have loaded windows from an earlier + // pilot for this measurement. If the freeze then fails, the sidecar + // must not record those as if they had come from this run. + let mut plugin = plugin_with_markers(); + plugin.pilot_windows = Some(( + PhaseWindow { start: 0.0, end: 0.2 }, + PhaseWindow { start: 0.5, end: 0.7 }, + )); + // No events => the fold carries no signal => the freeze cannot pick + // windows and must not leave the loaded ones in place. + assert!(plugin.camera_events.is_empty()); + plugin.freeze_pilot_windows(); + assert!( + plugin.pilot_windows.is_none(), + "stale pilot windows survived a failed freeze" + ); + assert!(!plugin.windows_are_frozen()); + } + #[test] fn press_latch_distinguishes_clicks_baselines_and_advances() { let mut latch = PressLatch::default(); @@ -3300,7 +3513,8 @@ mod tests { // marker validation rejects the fold, but the quicklook must fall back // to the free-running fold instead of blanking. let mut plugin = StageAA1Plugin { - valid_pixels: 10, + frame_width: 10, + frame_height: 1, camera_markers_us: vec![0, 1_000, 2_000, 10_000], ..StageAA1Plugin::default() }; From f22ce9a716fe823864e75e79277571ae059d742a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:12:58 +0200 Subject: [PATCH 25/30] =?UTF-8?q?docs(stage-a):=20=F0=9F=93=9D=20record=20?= =?UTF-8?q?the=20contrast-geometry=20decision=20as=20ADR=20012?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The photodiode's display mode used to select the optical geometry the published log-contrast was computed in, so a UI toggle changed a scientific quantity that A1's amplitude sweep settles against. ADR 012 records that geometry follows the bench, the measured dark level is applied to both sides of the complement (where it cancels), and a withheld `a` states its reason. Also documents the A1 `N_valid` definition (ROI area minus masked pixels, the same denominator `q_p` uses) and generalises the rule in architecture.md: a published field's meaning must not depend on the publisher's UI state. --- ...-contrast-geometry-is-bench-not-display.md | 82 +++++++++++++++++++ docs/architecture.md | 7 ++ docs/features/README.md | 2 +- docs/features/stage-a-a1.md | 14 +++- docs/features/stage-a-photodiode.md | 22 +++++ 5 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md diff --git a/docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md b/docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md new file mode 100644 index 0000000..7a39528 --- /dev/null +++ b/docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md @@ -0,0 +1,82 @@ +# ADR 012 — The contrast geometry follows the bench, not the display mode + +- **Status:** Accepted +- **Date:** 2026-07-27 +- **Relates to:** ADR 006 (two-plugin split), ADR 008 (optical waveform + inversion), ADR 010 (amplitude sweep), ADR 011 (Pockels transfer + calibration), + [Stage-A Photodiode](../features/stage-a-photodiode.md), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +The photodiode plugin has a display toggle: **RAW** plots the detector volts as +measured, **EXCITATION** plots `I_tot − I_pd`. `optical_summary` picked the +estimator's [`ContrastGeometry`] from that toggle — `Direct` under RAW, +`RejectedComplement` under EXCITATION — and published the result as +`PhotodiodeOpticalSummaryV1::measured_log_contrast`. + +That made a *published scientific quantity* depend on what the operator +happened to be looking at. It is wrong on the physics and it breaks A1: + +- On this bench the detector sits behind the PBS reject port and measures the + complement `I_pd = I_tot − I_exc`. That is settled by construction + (`knowledge base: setup/optical-path.md`), not a display choice. Under RAW the + published value was `ln(I_pd,max / I_pd,min)` — the *detector* contrast, not + the excitation contrast `a` that every A1 estimand is defined against. +- RAW is the default. A1's amplitude sweep settles `measured_a` against a target + `a` (`drive_sweep`): with the display left on its default the sweep compares + the wrong quantity, never settles, times out at 30 s per point, and writes a + wrong `measured_a` into every sweep sidecar. + +The same function also passed `dark_volts: 0.0` and a raw `reference_volts` +anchor, i.e. it dark-corrected one side of the complement and not the other. + +## Decision + +### 1. Geometry is a property of the optical configuration + +`optical_summary` always uses `ContrastGeometry::RejectedComplement`, anchored on +`reference_volts`. `measured_log_contrast` is always the excitation contrast. +The display `Mode` is presentational and never reaches the estimator; the status +readout is labelled `a (excitation)` unconditionally. + +If a future bench puts the detector in the excitation path, that is a new +optical configuration ID and a code change here — not a UI toggle. + +### 2. The dark level is measured, and applied to both sides + +`dark_volts` is a plugin setting with a **Capture dark** action (block the beam, +press; the mean of the current ring becomes the dark level, refused if it is not +below the `I_tot` reference). It is applied to the detector samples *and* +subtracted from the `reference_volts` anchor. + +Applied consistently, the DC dark term **cancels** out of the complement — the +excitation is a difference of two readings from the same DC-coupled detector, so +a common offset drops out. Correcting only one side is what would bias `a`, and +that is what the code did. `dark_id` reports `dark-measured` or `dark-none` so a +consumer can tell a real dark measurement from the un-measured default. + +### 3. A withheld `a` states its reason + +The estimator is deliberately fail-closed (clipping, no headroom, anchor below +signal). Those refusals now surface in the status readout as +`a unavailable: ` instead of the row silently disappearing. This matters +more under the new geometry: with an un-measured anchor left at ADC full scale, +`TotalPowerBelowSignal` is the expected first-run outcome, and the operator has +to be told to set `reference_volts`. + +## Consequences + +- `measured_log_contrast` is comparable across runs and independent of operator + UI state; A1's sweep settles against the quantity it targets. +- Runs recorded before this change that were taken with the display on RAW + carry a detector contrast in `measured_a`. They are distinguishable: their + sidecar has `anchor_id: "detector-direct"`. Those points must not be mixed + with `reference-volts` points. +- `excitation_headroom_volts` is, by construction, equal to + `excitation_min_volts` (both geometries are dark-referenced). The field is + kept because the contract publishes it, and is now documented as redundant + rather than silently duplicated. +- First use on a fresh bench requires setting `reference_volts` before any `a` + is published at all. This is intended: a wrong `a` is worse than no `a`. diff --git a/docs/architecture.md b/docs/architecture.md index dcba238..a46c4c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,6 +93,13 @@ calibration reads photodiode levels this way while driving only its own DAC: [`docs/adr/011-stage-a-pockels-transfer-calibration.md`](./adr/011-stage-a-pockels-transfer-calibration.md). Reserve the leased service path for *commanding* hardware someone else owns. +A published field is part of that contract, so its **meaning must not depend on +the publisher's UI state**. The photodiode plugin's display toggle used to +select the optical geometry the published log-contrast was computed in, which +silently retargeted A1's amplitude sweep whenever the chart was left on its +default. Geometry follows the bench, not the display: +[`docs/adr/012-stage-a-contrast-geometry-is-bench-not-display.md`](./adr/012-stage-a-contrast-geometry-is-bench-not-display.md). + ## Host Views Plugins declare host-rendered datasets and views through: diff --git a/docs/features/README.md b/docs/features/README.md index 2ff4830..56f1ed2 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -8,7 +8,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC so the *optical* output is a log- or linear-intensity sine, inverting the Pockels `sin²` transfer from settable `V_null`/`Vπ`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`Vπ` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd`, plus the geometry-corrected optical depth `a`. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 0b72be2..edb0405 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -133,8 +133,14 @@ quicklook falls back to the free-running fold on `T` instead of going empty. events per valid pixel in the trailing half-cycle, ON and OFF. A live indicator: are events appearing, does the ON/OFF timing look sane, is the response - saturating? It counts *every* event, so a noisy pixel weighs heavily — it is a - quicklook, not the response metric. + saturating? It counts *every* event in the ROI, so a noisy pixel weighs heavily + — it is a quicklook, not the response metric. + + `N_valid` is **ROI area minus masked pixels**, the same denominator `q_p` uses, + and the numerator counts only events inside that same region. The two are shown + side by side and have to mean the same thing; normalising `S_p` over the whole + sensor under-reported it by the ROI/frame ratio while counting events from + outside the ROI. 2. **Response probability** `q_p` @@ -186,10 +192,10 @@ still reset everything. | Input | Source | |---|---| -| camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()` | +| camera events, valid pixels | retained **EventStore** over a trailing analysis window; falls back to `frame.events()`, trimmed to the same window | | phase-0 markers | rising `frame.external_triggers()` — the host **banks trigger edges from dropped preview frames** into the next processed frame (drain-to-newest and the preview throttle drop whole frames; at low modulation frequencies the survivors alone rarely held 2 markers inside the analysis window) | | modulation period `T` | measured from the `EXT_TRIGGER` marker spacing; else the modulation plugin's acknowledged waveform — which, since the board-echo fallback, includes the **operator-armed UI drive**, not only service-path (leased) targets | -| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) | +| optical modulation depth `a` | photodiode plugin's optical summary (`measured_log_contrast`) — always the *excitation* contrast, independent of that plugin's display mode (ADR 012) | | ROI, masked pixels | augur-rs camera config (`CTX_GLOBAL_SETTINGS`) | ## Tests diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 122e6ff..07cb165 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -28,11 +28,33 @@ source (the camera `EXT_TRIGGER` belongs to A1's camera-clock analysis, not here ## Modes +The mode is a **display** choice only. It selects what the chart and the sample readout show; it +never changes a published quantity (ADR 012). + - **RAW** — ADC code and volts (`V = code · 3.3 / 4095`). - **EXCITATION** — the diode sits behind the PBS in the excitation path and measures the light removed from the beam (`I_pd = I_tot − I_exc`), so the plugin inverts against the user-set reference: `I_exc = I_tot − I_pd`, with `I_tot` given in photodiode volts. +## Optical log-contrast `a` + +`measured_log_contrast` in the published `PhotodiodeOpticalSummaryV1` is **always** the excitation +contrast `a = ln(I_exc,max / I_exc,min)`, in **both** display modes. The detector sits behind the +PBS reject port and measures the complement — that is a property of the bench, not of the display — +so the estimator always runs the `RejectedComplement` geometry against `reference_volts`. A1's +amplitude sweep settles on this value, so a display toggle must not be able to move it (ADR 012). + +- **Reference I_tot** (`reference_volts`) is the total-power anchor: the PD reading with the full + beam diverted into the diode. Until it is set to a real measurement, `a` is withheld. +- **Dark level** (`dark_volts`) + the **Capture dark** button: block the beam and press; the mean of + the current cache becomes the dark level. It is applied to the detector samples *and* to the + `I_tot` anchor, so it cancels out of the complement rather than biasing `a` — its job is to keep + the two sides consistent and to record the calibration the reading was taken under. `dark_id` in + the sidecar reads `dark-measured` or `dark-none` accordingly. +- The estimator is **fail-closed**: it refuses on ADC clipping, on no headroom above dark, and when + the anchor is not above the measured signal. A refusal is shown as `a unavailable: ` + rather than a missing row — a wrong `a` is worse than no `a`. + ## Chart - The visible window is decimated into at most 1 000 buckets; when a bucket covers more than one From 361fc676f4c22eb177ffbf8ade270e1e3f92e13c Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 20:41:43 +0200 Subject: [PATCH 26/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20bring=20the?= =?UTF-8?q?=20a=E2=82=80=20depth=20lock=20onto=20the=20fixed=20contrast=20?= =?UTF-8?q?geometry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the per-frequency exact-event-count depth lock (ADR 013, renumbered from 012 to clear the contrast-geometry ADR) onto the branch that carries the photodiode and modulation fixes, and repairs what that combination exposes. The lock closes `a_cmd ← a_cmd · a₀/a_measured` against the photodiode-measured log contrast. On its own branch that value's geometry followed the photodiode's *display toggle*, so under the default RAW mode it was the reject-port detector contrast rather than the excitation contrast — every locked depth would have been wrong. It is correct only together with the fixed geometry. Then the measurement itself: `a = ln(I_exc,max / I_exc,min)` is peak-to-peak, but the photodiode estimated it over a fixed 16 384-sample window — 0.82 s at 20 kSa/s, less than one cycle for every f < 1.2 Hz. Below one cycle the robust extrema see an arc of the waveform, so `a` comes out low and phase-dependent. That is exactly the sub-hertz plateau reference the A1 protocol normalises |H(f)| against, and the lock *divides* by it: a truncated estimate does not add noise, it drives the commanded depth up trial after trial until it rails at 6.0 or the detector clips. The contrast window is now sized from the phase-0 markers to cover CONTRAST_WINDOW_CYCLES whole cycles, floored at the old fixed window and capped by what the ring retains, and `a` is withheld outright below one cycle. The retained markers cannot measure a period longer than the ring — once the ring holds under a cycle it holds at most one marker — so the interval is remembered as markers go past instead of recovered from what survived eviction. `window_seconds` and `covered_cycles` join the optical summary (additive in V1). In the lock: - Find a₀ refuses up front when the published window is under one cycle at the current frequency, naming the cache length to raise. A1 always knows f, so this also covers an owner whose own markers cannot prove it. - the per-trial dwell is at least one estimator window, so a trial cannot average the depth it just replaced; the deadline grows with it - readings are spaced by half a window instead of per service_revision. Consecutive revisions share nearly their whole window, so three of them said no more than one - the trial value is the median and the spread is a stability gate: readings straddling a₀ abort the lock instead of locking onto a drifting drive - the clip warning threshold sits below the estimator's own refusal, where it can actually fire, instead of above it where it never could - a lock-table save failure is appended to the result instead of being overwritten by it --- .../013-stage-a-a1-event-count-depth-lock.md | 120 + docs/features/README.md | 1 + docs/features/stage-a-a1-automation.md | 11 + docs/features/stage-a-a1-event-count.md | 168 ++ docs/features/stage-a-a1.md | 20 +- plugins/stage-a-a1/README.md | 26 +- plugins/stage-a-a1/src/runtime.rs | 2167 +++++++++++++++-- plugins/stage-a-modulation/src/lib.rs | 5 +- plugins/stage-a-photodiode/src/lib.rs | 235 +- stage-a-io/src/estimator.rs | 20 + stage-a-plugin-contract/src/lib.rs | 12 + 11 files changed, 2577 insertions(+), 208 deletions(-) create mode 100644 docs/adr/013-stage-a-a1-event-count-depth-lock.md create mode 100644 docs/features/stage-a-a1-event-count.md diff --git a/docs/adr/013-stage-a-a1-event-count-depth-lock.md b/docs/adr/013-stage-a-a1-event-count-depth-lock.md new file mode 100644 index 0000000..7ee1c2e --- /dev/null +++ b/docs/adr/013-stage-a-a1-event-count-depth-lock.md @@ -0,0 +1,120 @@ +# ADR 013 — Stage-A A1 exact-event-count depth lock (`a₀`) + +- **Status:** accepted (2026-07-25) +- **Relates to:** ADR 009 (recording coordinator), ADR 010 (amplitude sweep via + leased `SetOpticalDepth`), ADR 011 (measured Pockels transfer calibration), + ADR 012 (the contrast geometry the measured `a` comes from), + [Stage-A A1 Exact Event Count](../features/stage-a-a1-event-count.md) + +## Context + +The minimum-depth workflow sweeps the depth `a` at one frequency and fits `a50`. +The **exact-event-count** workflow is the complement: freeze **one** depth + +```math +a_0=\ln\!\left(\frac{I_{\mathrm{exc,max}}}{I_{\mathrm{exc,min}}}\right), +\qquad I_\mathrm{exc}=I_\mathrm{tot}-I_\mathrm{pd} +``` + +and hold **that measured value** constant while the frequency varies, so the +event count per half-cycle is compared across `f` at equal optical contrast. + +`a₀` is defined on the **photodiode-measured** log contrast, never on a DAC +excursion. That is exactly where the existing sweep path stops short: ADR 010 +drives `SetOpticalDepth { depth_a_milli }` **open-loop**, trusting the measured +Pockels inversion (ADR 011) to turn a commanded depth into an optical one, and +then only *waits* for the measured `a` to arrive. The inversion is static, so as +`f` rises the drive electronics and the crystal response roll off and the +delivered depth falls short of the commanded one. Waiting cannot fix a +systematic gain error: the sweep would hit its 30 s settle cap and record a +point at the wrong depth (with the measured value honestly in the sidecar, but +the run wasted). + +A second, subtler problem: ADR 010 already notes that "the operator's own +`depth a` re-applies on the next modulation settings sync after release". A +depth found in one operator action and recorded in a *later* one can therefore be +silently overwritten between the two. + +## Decision + +**1. A closed-loop `a₀` lock in A1, separate from recording.** A *Find a₀* +button runs a small state machine — `AcquiringLease → (per trial) SettingDepth → +Measuring → …release` — that iterates + +```math +a_\text{cmd} \leftarrow a_\text{cmd}\cdot\frac{a_0}{a_\text{measured}} +``` + +until the photodiode-measured `a` is within an absolute tolerance of `a₀` +(default ±0.02), at most 8 trials, each correction capped at ×2/÷2 and clamped +to the owner's `0.01..=6.0`. The delivered depth is proportional to the commanded +one to first order, so this converges in two or three trials while absorbing +whatever roll-off the frequency introduces. It reuses the ADR 010 contract +command unchanged — no new modulation command, no optical math outside its owner. + +The lock **records nothing** and releases the lease with `safe_off = false`, so +the drive stays exactly where the lock left it. + +Measurement hygiene: readings are only taken after the operator's settle dwell +has passed, and one reading per **fresh** photodiode `service_revision` (three +per trial), so a slow publisher is not averaged once per control tick. A +measured `a ≤ 0`, a missing optical summary, or an owner rejection ends the lock +with the owner's own wording — a refused depth *is* the "`a₀` unreachable at this +operating point" answer. + +**2. The result is data, not a transient.** Each finished lock is stored as one +row per frequency — `frequency_hz`, `target_a`, `commanded_a`, `measured_a`, +`trials`, `converged`, clip fractions, timestamp — replacing any earlier row +within 1 % of the same frequency, shown in an `a₀ lock table` host view, and +mirrored to `a0_locks.json` in the output folder so the found depths survive a +restart and can be cited offline. Non-converged attempts are kept for the record +but never arm a recording. + +**3. Recording replays the locked depth under the lease.** *Record a₀ point* +does **not** simply record at whatever the drive currently is. It runs the ADR 010 +sweep machinery as a **one-point sweep of a new kind**: lease → command the +locked `a_cmd` → confirm the measured `a` holds `a₀` within the lock tolerance → +record through the unchanged coordinator → release. This gives three things at +once: the depth is re-asserted (immune to an intervening settings sync), the +lease locks the operator's modulation settings out for the whole point, so +"never change amplitude during the recorded interval" is enforced rather than +trusted, and the point is one button press. + +To express this, a sweep point became a pair — what the drive is **commanded** +to, and the depth it is **expected to measure**. The amplitude sweep sets both +equal (it trusts the calibration); an event-count point deliberately does not, +and the difference *is* the absorbed roll-off. + +**4. Naming and provenance.** Event-count points take the role suffix `_ec` and +carry their **frequency** in the stem (`…_ec_f50Hz`, `…_ec_f0p5Hz`) instead of a +sweep-point index, because one measurement id spans the whole frequency sweep at +the single frozen depth. The sidecar gains `sweep.commanded_a` and an +`[a0_lock]` section (target, commanded, measured-at-lock, frequency-at-lock, +trials, converged, locked-at), and both recorders' own sidecars carry the same +values as string metadata. + +**5. What stays the operator's.** The flux point, camera configuration, ROI/mask, +pedestal, bias set, gates, reference epoch, the frequency itself, the +`I_tot` anchor, the zero-depth background and the pilot (already separate +buttons), the randomised frequency order, the interleaved low-frequency +reference, and the repeated blocks. A1 adds exactly two buttons per frequency — +*Find a₀* and *Record a₀ point* — because the protocol's ordering and +randomisation decisions are scientific, not mechanical. + +## Consequences + +- A1's scoped hardware reach is unchanged in kind (still only the armed drive's + depth, still only while leased) but now closed-loop: it reads the photodiode + to decide what to command. +- The recorded amplitude is provably the measured `a₀`, not a calibrated guess, + at every frequency — including frequencies where the static Pockels inversion + is no longer accurate. +- `a₀` itself is **not** frozen numerically in this repository: it is an operator + input, to be chosen from the low-frequency scout (several events per + pixel-half-cycle, still proportional, refractory-safe at the top frequency). + The plugin default is a placeholder. +- The refractory condition `2 f a₀/C ≪ 1/τ_refr` is *not* checked in the plugin; + it is a choice made once when `a₀` is picked, and stays with the operator. +- Re-locking after changing the flux point, the calibration or `a₀` is required: + a stored lock is only armed for a matching frequency **and** a matching `a₀`, + and *Clear a₀ lock table* exists for the rest. diff --git a/docs/features/README.md b/docs/features/README.md index 56f1ed2..8606c60 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -11,6 +11,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. +- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀`, a per-frequency lock table on disk, and a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-automation.md b/docs/features/stage-a-a1-automation.md index 529a10d..bcbeafc 100644 --- a/docs/features/stage-a-a1-automation.md +++ b/docs/features/stage-a-a1-automation.md @@ -25,6 +25,17 @@ > §3 coordinator with `sweep.requested_a` / `point_index` / `point_total` in > the sidecar. Remaining below: scout/randomized order, multi-`f`/`I_k` > iteration, the `UNIDENTIFIABLE` rule, and the `a50` fit. +> +> **Update (2026-07-25):** the *second* workflow — **exact event count**, holding +> one measured depth `a₀` across the frequency sweep — now has its own blocks +> (ADR 013, [brief](./stage-a-a1-event-count.md)): a closed-loop **Find a₀** per +> frequency (§1–§2 applied the other way round — measure, then correct the +> *commanded* depth) and a **Record a₀ point** that replays the locked depth under +> the lease through the §3 coordinator. The multi-`f` iteration below stays +> deliberately manual there: randomising the frequency order, interleaving a +> low-frequency reference and repeating independent blocks are scientific ordering +> decisions, so A1 exposes them as per-frequency button presses rather than one +> opaque run. ## Goal diff --git a/docs/features/stage-a-a1-event-count.md b/docs/features/stage-a-a1-event-count.md new file mode 100644 index 0000000..f9c4d7e --- /dev/null +++ b/docs/features/stage-a-a1-event-count.md @@ -0,0 +1,168 @@ +# Stage-A A1 Exact Event Count — the `a₀` depth lock + +- **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) +- **Status:** Built — per-frequency `a₀` lock + one-button event-count point +- **Design:** [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md); builds + on [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (leased + `SetOpticalDepth`) and [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) + (the RAW + PDQ + sidecar coordinator) +- **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), + [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md), + [Stage-A Photodiode](./stage-a-photodiode.md) + +## Purpose + +The minimum-depth workflow sweeps `a` at one frequency to fit `a50`. The +**exact-event-count** workflow is the complement: freeze **one** depth + +```math +a_0=\ln\!\left(\frac{I_{\mathrm{exc,max}}}{I_{\mathrm{exc,min}}}\right), +\qquad I_\mathrm{exc}=I_\mathrm{tot}-I_\mathrm{pd} +``` + +and hold that **photodiode-measured** value constant while the frequency varies, +so event counts per half-cycle are comparable across `f` at equal optical +contrast. `a₀` is a measured log contrast — **never** a DAC-code excursion. + +## Why a lock is needed at all + +`ModulationCommandV1::SetOpticalDepth` commands a depth through the *measured* +Pockels inversion (`V_null`, `Vπ`, `u_k` — see the calibration brief). That +inversion is static, so at higher frequencies the drive electronics and crystal +response roll off and the delivered optical depth falls short of the commanded +one. The amplitude sweep (ADR 010) only *waits* for the measured `a`, which +cannot correct a systematic gain error — it would hit the 30 s settle cap and +record at the wrong depth. + +The lock closes that loop: it commands, measures, and corrects until the +photodiode reports `a₀`. + +## The workflow, one frequency at a time + +Everything up to the references is unchanged and stays the operator's: freeze the +flux point and camera configuration, reuse the same film position, ROI/mask, +optical pedestal, bias set, gates and reference epoch as the minimum-depth +measurement, keep ON and OFF separate, and per frequency record the full-extinction +`I_tot` anchor, the zero-depth background and the high non-saturating pilot (the +existing **Record pilot** / **Record background** buttons; background reuse +across frequencies is not automated, i.e. off by default). Then: + +1. Set the frequency `f` in the modulation plugin (yours — the drive is armed + there, A1 only reads it). +2. Enter **a₀** once for the whole sweep, and press **Find a₀**. A1 leases the + modulation owner and trims the commanded depth until the photodiode measures + `a₀` at *this* frequency. Nothing is recorded; the drive is left at the depth + it found and the result is stored for `f`. +3. Press **Record a₀ point (event-count)**. A1 re-applies the found depth under a + modulation lease, waits for the measured `a` to hold `a₀`, and records one + atomic camera RAW + photodiode PDQ + sidecar under one run id. +4. Repeat for the next frequency. Randomising the frequency order, interleaving + the low-frequency reference and repeating independent blocks (three where + practical) are yours — every point is one button press. + +## Controls + +| Control | Meaning | +|---|---| +| a₀ (measured log contrast) | the one photodiode-measured depth held across the whole frequency sweep | +| a₀ tolerance (absolute) | convergence band on `|measured a − a₀|`; also the settle band an event-count point must hold before it records (default ±0.02) | +| Find a₀ (lock the drive depth) | closed-loop trim of the commanded depth at the current frequency; records nothing, stores the result, leaves the drive there | +| Record a₀ point (event-count) | re-applies the locked depth under the lease and records one atomic frequency point (`…_ec_fHz`) | +| Clear a₀ lock table | drops every stored lock and rewrites `a0_locks.json` | +| Stop (abort recording / sweep) | also aborts a running lock | + +The **Sweep settle (s)** value in the Recording section is reused as the +per-trial dwell before the lock starts averaging. + +## The lock loop + +```math +a_\text{cmd} \leftarrow a_\text{cmd}\cdot\frac{a_0}{a_\text{measured}} +``` + +- Starts from an earlier lock at the same frequency when one exists, otherwise + from `a₀` itself (the calibrated open-loop guess). +- Converges when `|measured − a₀| ≤ tolerance`; at most **8 trials**, each + correction capped at ×2/÷2 and clamped to the owner's `0.01..=6.0`. +- Per trial it waits the settle dwell, then averages **three fresh** photodiode + optical summaries (one per new `service_revision`, so a slow publisher is not + averaged once per control tick); it evaluates early with fewer readings only if + the 30 s measurement deadline hits first. +- Ends with the owner's own wording when a commanded depth is **rejected** (lobe + ceiling, DAC limit) — that is the "`a₀` is unreachable at this operating point, + lower `a₀` or `I_k`" answer — or reports the drivable limit when the correction + rails at `0.01`/`6.0`. +- Photodiode clipping above 1 % is called out in the result message and stored + with the lock: a clipped window makes the measured `a` a truncated estimate. +- Releases the lease with `safe_off = false`, so the drive holds the found depth. + +## The lock table + +One row per frequency (a re-lock within 1 % of a stored frequency replaces it): +frequency, target `a₀`, commanded `a`, measured `a`, trials, state, locked-at. +Visible as the **A1 a₀ locks** host view and mirrored to +`/a0_locks.json`, so the found depths survive a restart and can be +cited offline. A non-converged row is kept for the record but **never** arms a +recording; a stored lock only arms an event-count point when both its frequency +**and** its `a₀` still match the current settings. + +## Why recording re-applies the depth + +*Record a₀ point* does not simply record at whatever the drive currently is. It +runs the ADR 010 sweep machinery as a one-point sweep of a new kind — lease → +command the locked depth → confirm the measured `a` holds `a₀` → record → release +— which buys three things: + +- the depth is **re-asserted**, so an intervening modulation settings sync (which + re-applies the operator's own `depth a`) cannot silently spoil the point; +- the lease **locks the operator's modulation settings out** for the whole point, + so *"never change amplitude during the recorded interval"* is enforced rather + than trusted; +- it stays one button press. + +Internally a sweep point is now a pair: the depth the drive is **commanded** to +and the depth it is **expected to measure**. The amplitude sweep sets both equal; +an event-count point deliberately does not, and the difference is the roll-off +the lock absorbed. + +## Naming and sidecar + +Event-count points use the role suffix `_ec` and carry the **frequency** in the +stem instead of a sweep-point index — one measurement id spans the whole +frequency sweep at the single frozen depth: + +- `/__ec_f50Hz.raw` (+ the host's own `.toml`) +- `/__ec_f50Hz_pd.pdq` + `_pd.json` +- `/__ec_f50Hz_config.toml` + +Sub-hertz frequencies keep the decimal as `p` (`f0p5Hz`). The A1 sidecar adds +`sweep.commanded_a` and an `[a0_lock]` section (`target_a`, `commanded_a`, +`measured_a_at_lock`, `frequency_hz_at_lock`, `trials`, `converged`, +`locked_at_utc`); both recorders' own sidecars carry `a0_target`, +`a0_commanded_a`, `a0_lock_measured_a` and `a0_lock_frequency_hz` as metadata. +The measured `a` of the recording itself stays in `[optical]` as for every run. + +## Choosing `a₀` (still an operator decision) + +No numerical `a₀` is frozen in this repository — the plugin default is a +placeholder. Pick it from the low-frequency scout so that + +- the low-frequency response gives **several** events, not the one-event floor; +- the event count is still **proportional** to depth and has not saturated; +- the refractory condition `2 f a₀/C ≪ 1/τ_refr` holds at the **highest** + frequency (checked once by you when picking `a₀`; the plugin does not test it); +- the same measured `a₀` is **reachable at every frequency** in the sweep — the + lock reports when it is not, before any data is recorded. + +Today's low-frequency `a50` result is a sensible starting point; targeting +several plateau events per pixel per half-cycle is a good scout criterion. + +## Tests + +`cargo test -p augur-plugin-stage-a-a1` covers the lock converging against a +simulated 60 %-gain bench (and leaving the drive at the found depth with a +`safe_off = false` release), the unreachable-depth case railing at the drive +limit without arming a recording, an owner rejection surfacing verbatim, an +event-count point commanding the **locked** depth rather than `a₀`, the +`_ec_fHz` stem and `[a0_lock]` sidecar section, file-safe frequency tags, and +the one-row-per-frequency lock table round-tripping through `a0_locks.json`. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index edb0405..dec9534 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -1,11 +1,15 @@ # Stage-A A1 Analysis - **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) -- **Status:** Recording coordinator + live quicklooks + amplitude sweep +- **Status:** Recording coordinator + live quicklooks + amplitude sweep + `a₀` lock - **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button - press forwarding) + press forwarding), + [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count + `a₀` lock) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) +- **Second workflow:** [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) + — hold one *measured* depth `a₀` across the frequency sweep ## Purpose @@ -43,6 +47,7 @@ folder. A1 makes each recording one button press: | Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | | Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | | Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | +| a₀ / Find a₀ / Record a₀ point | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep — see [its brief](./stage-a-a1-event-count.md) | The record and sweep buttons stay **disabled until an output folder is selected**. @@ -63,10 +68,12 @@ require `min a > 0` — record `a≈0` with the background button instead. Sidec of sweep recordings additionally carry `sweep.requested_a`, `sweep.point_index` and `sweep.point_total`. After the sweep releases the lease, the drive holds the last sweep amplitude until the operator's own `depth a` setting is re-applied -(any modulation settings change re-sends it). +(any modulation settings change re-sends it) — which is exactly why an +event-count point re-applies its locked depth under the lease instead of trusting +the drive to still be where a previous action left it (ADR 013). **Naming.** Files share an `_[_role]` stem under an `/` subfolder -(`_pilot` / `_background` tag the reference runs): +(`_pilot` / `_background` tag the reference runs, `_ec_fHz` an event-count point): - `/_.raw` — camera RAW, under the **host output root**, with the host's own `.toml` sidecar (camera biases, ROI) written next to it. @@ -206,5 +213,6 @@ path, file-safe id generation, UTC timestamp formatting, the config-sidecar buil the pilot-window round-trip through the measurement folder, press-latch edge/baseline semantics, the jittery-marker free-running fallback, sweep-point spacing, the sweep-point sidecar fields, the ordered camera → PDQ → PDQ finalize → camera -finalize lifecycle (including envelope identity/revision and save location), and -the selective discontinuity reset. +finalize lifecycle (including envelope identity/revision and save location), the +selective discontinuity reset, and the `a₀`-lock set listed in the +[exact-event-count brief](./stage-a-a1-event-count.md). diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 50d98a6..4ab0638 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -24,6 +24,28 @@ optical drive in the modulation plugin; A1 only reads its published settings. one normal recording. Sidecars carry `sweep.requested_a` / `point_index` / `point_total`. - The record/sweep buttons are disabled until an output folder is selected. +## Exact event count (`a₀` lock) + +The second Stage-A workflow holds **one** photodiode-measured depth +`a₀ = ln(I_exc,max / I_exc,min)` constant while the frequency varies. Because the measured Pockels +inversion is static, the delivered depth rolls off with frequency — so the depth must be found by +measurement, not calculated. + +- **a₀** / **a₀ tolerance** — the frozen measured depth and its convergence band (default ±0.02). +- **Find a₀** — per frequency: leases the modulation owner and iterates + `commanded a ← commanded a · a₀/measured a` (≤ 8 trials, averaging three fresh photodiode summaries + per trial after **Sweep settle (s)**) until the photodiode measures `a₀`. Records nothing, leaves the + drive at the depth it found, and stores one row per frequency — shown in the **A1 a₀ locks** view and + mirrored to `a0_locks.json`. An unreachable `a₀` is reported (drive limit or the owner's own + rejection) before any data is recorded. +- **Record a₀ point (event-count)** — re-applies the locked depth under the lease (so the amplitude + cannot change during the recorded interval), waits for the measured `a` to hold `a₀`, and records + one atomic frequency point named `…_ec_fHz` with an `[a0_lock]` sidecar section. +- **Clear a₀ lock table** — after changing the flux point, the calibration or `a₀` itself. + +Frequency order, the interleaved low-frequency reference and the repeated blocks stay yours — every +point is one button press. + Files share an `_` stem: `/_.raw` (camera, under the host output root), `/__pd.pdq` + `.json` (photodiode, under its data root), and `/__config.toml` (A1, under the chosen folder). Point all three roots at the same @@ -48,6 +70,8 @@ the frequency), falling back to the modulation plugin's acknowledged waveform. T pixels come from the augur-rs camera config. See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the full brief, -[ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, and +[ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, +[docs/features/stage-a-a1-event-count.md](../../docs/features/stage-a-a1-event-count.md) plus +[ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 64ca65b..e32636e 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -14,7 +14,12 @@ //! settings into the sidecar. The **amplitude sweep** (ADR 010) is the one scoped //! exception: per sweep point it retargets the armed drive's *depth* through the //! leased modulation service (`SetOpticalDepth`), waits for the photodiode-measured -//! `a` to settle, and records the point through the same coordinator. +//! `a` to settle, and records the point through the same coordinator. The +//! **exact-event-count workflow** (ADR 013) reuses that path the other way round: +//! the `a₀` **lock** trims the *commanded* depth closed-loop until the photodiode +//! *measures* the one frozen depth `a₀`, and an **event-count point** replays that +//! trimmed depth under the same lease so one atomic frequency point is recorded at +//! exactly `a₀`. //! //! 2. **Live sanity quicklooks.** Folding the camera event stream on the modulation //! period `T` (defined by the firmware phase-0 `EXT_TRIGGER`), it renders the @@ -66,6 +71,8 @@ const ROLLING_DATASET_ID: &str = "stage-a-a1.rolling-response"; const ROLLING_VIEW_ID: &str = "stage-a-a1.rolling-response.view"; const RESPONSE_CURVE_DATASET_ID: &str = "stage-a-a1.response-curve"; const RESPONSE_CURVE_VIEW_ID: &str = "stage-a-a1.response-curve.view"; +const A0_LOCK_DATASET_ID: &str = "stage-a-a1.a0-locks"; +const A0_LOCK_VIEW_ID: &str = "stage-a-a1.a0-locks.view"; /// Camera events retained for the live fold. At the bench event rates this is a /// few seconds of history and keeps the fold cost bounded. @@ -85,11 +92,84 @@ const MAX_MARKERS: usize = 65_536; /// after this long and record anyway (the sidecar stores the measured value). const SWEEP_SETTLE_TIMEOUT_MS: u64 = 30_000; +/// Closed-loop trials the `a₀` lock spends on one frequency before it gives up +/// and reports the best commanded depth it reached. +const A0_LOCK_MAX_TRIALS: u32 = 8; +/// Per-trial cap on the multiplicative correction of the commanded depth, so one +/// noisy photodiode reading cannot slam the drive across its whole range. +const A0_LOCK_MAX_STEP_RATIO: f64 = 2.0; +/// Independent photodiode readings taken per trial (fewer only when the +/// measurement deadline hits first). Their *median* is the trial's value and +/// their spread is the stability check — one estimator window already averages +/// many cycles, so repeating it is about catching drift, not reducing noise. +const A0_LOCK_SAMPLES: usize = 3; +/// Fraction of one estimator window that must pass between two readings for +/// them to count as independent. Consecutive `service_revision`s share almost +/// their whole window, so sampling per revision alone measures the publisher's +/// tick rate rather than the drive. +const A0_LOCK_SAMPLE_SPACING: f64 = 0.5; +/// Spread across a trial's readings, relative to its tolerance, above which the +/// operating point is called unstable instead of locked. A drifting `a` that +/// happens to cross the target on one reading is not a lock. +const A0_LOCK_MAX_SPREAD_TOLERANCES: f64 = 2.0; +/// Clipping fraction above which a lock's measured `a` is called out as +/// unreliable in the operator message. +/// +/// Deliberately far below the estimator's own `MAX_CLIP_FRACTION` (1 ‰, above +/// which it withholds `a` altogether): a threshold at or above that one could +/// never fire, because a published summary has already passed it. +const A0_LOCK_CLIP_WARNING: f64 = 0.000_2; +/// Closed range of commanded optical depths the modulation owner accepts. +const COMMANDED_A_MIN: f64 = 0.01; +const COMMANDED_A_MAX: f64 = 6.0; +/// Relative distance within which two frequencies are the same sweep point. +const FREQUENCY_MATCH_FRACTION: f64 = 0.01; +/// Lock table persisted in the output folder, so found depths survive a restart. +const A0_LOCK_FILE: &str = "a0_locks.json"; + /// Absolute/relative tolerance for "the measured `a` reached the sweep target". fn sweep_tolerance(target_a: f64) -> f64 { (target_a * 0.10).max(0.05) } +/// Commanded optical depth clamped to what the modulation owner accepts. +fn clamp_commanded_a(depth_a: f64) -> f64 { + if depth_a.is_finite() { + depth_a.clamp(COMMANDED_A_MIN, COMMANDED_A_MAX) + } else { + COMMANDED_A_MIN + } +} + +/// Wire encoding of a commanded optical depth for `SetOpticalDepth`. +fn depth_a_milli(depth_a: f64) -> u32 { + (depth_a * 1_000.0).round().clamp(0.0, u32::MAX as f64) as u32 +} + +/// Whether two frequencies name the same sweep point (drive vs trigger readback +/// never agree to the last digit). +fn same_frequency(left: f64, right: f64) -> bool { + let scale = left.abs().max(right.abs()); + (left - right).abs() <= (scale * FREQUENCY_MATCH_FRACTION).max(1e-6) +} + +fn frequency_label(hz: f64) -> String { + format!("{hz:.3} Hz") +} + +/// Compact file-safe frequency tag for an event-count point's stem: +/// `50 Hz → f50Hz`, `0.5 Hz → f0p5Hz`. +fn frequency_tag(hz: f64) -> String { + let mut text = format!("{hz:.3}"); + while text.ends_with('0') { + text.pop(); + } + if text.ends_with('.') { + text.pop(); + } + format!("f{}Hz", text.replace('.', "p")) +} + trait RecordingControl { fn request_service(&mut self, request: &PluginServiceRequest); fn request_host(&mut self, request: &HostCommandRequest); @@ -177,6 +257,9 @@ enum RecRole { Pilot, /// Unmodulated (`a≈0`) reference that gives the false-response floor. Background, + /// One atomic frequency point of the exact-event-count workflow, recorded at + /// the one frozen depth `a₀` the lock found for that frequency. + EventCount, } impl RecRole { @@ -186,6 +269,7 @@ impl RecRole { RecRole::Normal => "", RecRole::Pilot => "_pilot", RecRole::Background => "_background", + RecRole::EventCount => "_ec", } } @@ -194,6 +278,7 @@ impl RecRole { RecRole::Normal => "point", RecRole::Pilot => "pilot", RecRole::Background => "background", + RecRole::EventCount => "event-count point", } } } @@ -249,13 +334,47 @@ enum SweepPhase { Recording, } +/// What a leased sweep is for: the amplitude sweep of one `(I_k, f)` row, or one +/// atomic frequency point of the exact-event-count workflow. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SweepKind { + Amplitude, + EventCount, +} + +impl SweepKind { + fn role(self) -> RecRole { + match self { + SweepKind::Amplitude => RecRole::Normal, + SweepKind::EventCount => RecRole::EventCount, + } + } +} + +/// One sweep point: what the drive is *commanded* to, and the +/// photodiode-measured `a` that point is supposed to produce. +/// +/// The amplitude sweep asks for its own value open-loop, trusting the Pockels +/// calibration, so both are equal. An event-count point replays a commanded +/// depth the `a₀` lock already trimmed closed-loop against the *measured* depth, +/// so there its commanded depth is deliberately **not** the depth it expects to +/// measure — that difference is the drive roll-off the lock absorbed. +#[derive(Debug, Clone, Copy, PartialEq)] +struct SweepPoint { + commanded_a: f64, + expected_a: f64, +} + /// One "record every point of the amplitude range" run: per point the sweep /// retargets the leased modulation drive, waits for the photodiode-measured /// `a` to settle, and hands off to the normal recording coordinator. struct Sweep { phase: SweepPhase, - /// Requested `a` per point, ascending over `[min_a, max_a]`. - points: Vec, + kind: SweepKind, + /// The points to record, in order. + points: Vec, + /// The `a₀` lock an event-count point replays; `None` for the amplitude sweep. + lock: Option, index: usize, lease_id: LeaseId, lease_granted: bool, @@ -274,8 +393,21 @@ struct Sweep { } impl Sweep { + fn point(&self) -> SweepPoint { + self.points.get(self.index).copied().unwrap_or(SweepPoint { + commanded_a: 0.0, + expected_a: 0.0, + }) + } + + /// The photodiode-measured `a` this point must settle at. fn target_a(&self) -> f64 { - self.points.get(self.index).copied().unwrap_or(0.0) + self.point().expected_a + } + + /// The depth the drive is commanded to for this point. + fn commanded_a(&self) -> f64 { + self.point().commanded_a } fn total(&self) -> usize { @@ -283,6 +415,81 @@ impl Sweep { } } +/// Where the `a₀` lock is within its current closed-loop trial. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum A0LockPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// SetOpticalDepth for the current trial sent; waiting for Applied. + SettingDepth, + /// Settling, then averaging fresh photodiode readings for this trial. + Measuring, +} + +/// One "find the commanded depth that makes the photodiode measure `a₀` at this +/// frequency" run. Iterates `commanded ← commanded · a₀/measured` under a +/// modulation lease and never records anything itself. +struct A0Lock { + phase: A0LockPhase, + /// The photodiode-measured log contrast the operator froze for the sweep. + target_a: f64, + /// Convergence band on `|measured − target|`. + tolerance: f64, + /// The depth the current trial commands. + commanded_a: f64, + /// Frequency this lock belongs to, captured when it started. + frequency_hz: f64, + /// 1-based trial counter, bounded by `A0_LOCK_MAX_TRIALS`. + trial: u32, + /// Independent photodiode readings collected for the current trial. + samples: Vec, + /// `service_revision` of the newest photodiode summary already sampled, so a + /// slow publisher is not sampled once per control tick. + sampled_revision: Option, + /// Earliest instant the next reading may be taken: the settle dwell before + /// the first, then one sample spacing after each. + measure_from_ms: u64, + /// Estimator window length (ms) the photodiode reported when this trial + /// commanded its depth. Both the dwell and the sample spacing derive from + /// it, because a reading taken sooner still contains the previous depth. + window_ms: u64, + /// Give-up deadline for the current trial's measurement. + deadline_ms: u64, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + depth_req: u64, + depth_applied: bool, + last_activity_ms: u64, + stop_requested: bool, +} + +/// The result of one lock: the commanded depth that produced the frozen `a₀` at +/// one frequency. Persisted in `a0_locks.json` and replayed by event-count points. +#[derive(Debug, Clone, Serialize, serde::Deserialize)] +struct A0LockPoint { + frequency_hz: f64, + /// The frozen `a₀` the lock aimed at. + target_a: f64, + /// What the drive must be commanded to in order to *measure* `target_a`. + commanded_a: f64, + /// The photodiode-measured `a` averaged over the final trial. + measured_a: f64, + trials: u32, + /// False when the lock ran out of trials or hit a drive limit; such a row is + /// kept for the record but never arms an event-count recording. + converged: bool, + locked_at_unix_ms: u64, + low_clip_fraction: Option, + high_clip_fraction: Option, +} + +/// On-disk form of the per-frequency lock table. +#[derive(Debug, Clone, Default, Serialize, serde::Deserialize)] +struct A0LockTable { + locks: Vec, +} + pub struct StageAA1Plugin { enabled: bool, runtime_role: PluginRuntimeRole, @@ -346,6 +553,23 @@ pub struct StageAA1Plugin { /// Latched by the Start sweep button, consumed next control tick. sweep_pending: bool, sweep: Option, + // -- exact event-count depth a₀ (ADR 013) -- + /// The one photodiode-measured log contrast held across the frequency sweep. + a0_target: f64, + /// Convergence band on `|measured a − a₀|` for the lock and for an + /// event-count point's settle check. + a0_tolerance: f64, + /// Latched by the Find a₀ button, consumed next control tick. + a0_lock_pending: bool, + /// Latched by the Record a₀ point button, consumed next control tick. + a0_point_pending: bool, + a0_lock: Option, + /// One converged (or attempted) lock per frequency, newest per frequency + /// wins; mirrored to `a0_locks.json` in the output folder. + a0_locks: Vec, + /// Output folder the lock table was last read for, so it is re-read only + /// when the experiment folder changes. + loaded_locks_folder: Option, // -- momentary-button press forwarding (see PressLatch) -- press_start: PressLatch, press_pilot: PressLatch, @@ -355,6 +579,9 @@ pub struct StageAA1Plugin { press_clear: PressLatch, press_record_point: PressLatch, press_clear_curve: PressLatch, + press_find_a0: PressLatch, + press_record_a0: PressLatch, + press_clear_a0: PressLatch, } impl Default for StageAA1Plugin { @@ -395,6 +622,15 @@ impl Default for StageAA1Plugin { settle_s: 2.0, sweep_pending: false, sweep: None, + // No numerical a₀ is frozen in the repository: this default is a + // placeholder the operator replaces with the scout result. + a0_target: 0.5, + a0_tolerance: 0.02, + a0_lock_pending: false, + a0_point_pending: false, + a0_lock: None, + a0_locks: Vec::new(), + loaded_locks_folder: None, press_start: PressLatch::default(), press_pilot: PressLatch::default(), press_background: PressLatch::default(), @@ -403,6 +639,9 @@ impl Default for StageAA1Plugin { press_clear: PressLatch::default(), press_record_point: PressLatch::default(), press_clear_curve: PressLatch::default(), + press_find_a0: PressLatch::default(), + press_record_a0: PressLatch::default(), + press_clear_a0: PressLatch::default(), } } } @@ -506,6 +745,11 @@ impl StageAA1Plugin { (hz > 0.0).then(|| 1_000_000.0 / hz) } + /// Modulation frequency implied by [`Self::period_us`]. + fn frequency_hz(&self) -> Option { + self.period_us().map(|period| 1_000_000.0 / period) + } + /// Modulation period measured from the phase-0 markers (mean spacing). fn measured_period_us(&self) -> Option { if self.camera_markers_us.len() < 2 { @@ -996,9 +1240,25 @@ impl StageAA1Plugin { "sweep_requested_a".into(), format!("{:.6}", sweep.target_a()), ); + meta.insert( + "sweep_commanded_a".into(), + format!("{:.6}", sweep.commanded_a()), + ); meta.insert("sweep_point_index".into(), (sweep.index + 1).to_string()); meta.insert("sweep_point_total".into(), sweep.total().to_string()); } + if let Some(lock) = self.sweep.as_ref().and_then(|sweep| sweep.lock.as_ref()) { + meta.insert("a0_target".into(), format!("{:.6}", lock.target_a)); + meta.insert("a0_commanded_a".into(), format!("{:.6}", lock.commanded_a)); + meta.insert( + "a0_lock_measured_a".into(), + format!("{:.6}", lock.measured_a), + ); + meta.insert( + "a0_lock_frequency_hz".into(), + format!("{:.6}", lock.frequency_hz), + ); + } if let Some(a) = self.measured_a() { meta.insert("measured_a".into(), format!("{a:.6}")); } @@ -1037,12 +1297,26 @@ impl StageAA1Plugin { let now_ms = now_unix_ms(); let id = sanitize_stem(self.measurement_id.trim()); // Sweep points get a stable per-point tag so the row's files sort by - // sweep order as well as by timestamp. + // sweep order as well as by timestamp. Event-count points instead carry + // their frequency, because one measurement id spans the whole frequency + // sweep at the single frozen depth a₀. + let live_hz = self.frequency_hz(); let sweep_tag = self .sweep .as_ref() .filter(|sweep| sweep.phase == SweepPhase::Recording) - .map(|sweep| format!("_p{:02}", sweep.index + 1)) + .map(|sweep| match sweep.kind { + SweepKind::Amplitude => format!("_p{:02}", sweep.index + 1), + SweepKind::EventCount => { + let hz = sweep + .lock + .as_ref() + .map(|lock| lock.frequency_hz) + .or(live_hz) + .unwrap_or_default(); + format!("_{}", frequency_tag(hz)) + } + }) .unwrap_or_default(); let stem = format!( "{id}_{}{}{sweep_tag}", @@ -1069,7 +1343,8 @@ impl StageAA1Plugin { match role { RecRole::Pilot => self.freeze_pilot_windows(), RecRole::Background => self.capture_background_floor(), - RecRole::Normal => {} + // Both keep the row's pilot-frozen windows and background floor. + RecRole::Normal | RecRole::EventCount => {} } self.start_camera(context); @@ -1283,11 +1558,19 @@ impl StageAA1Plugin { } /// The requested `a` per sweep point, ascending and inclusive of both ends. - fn sweep_points(&self) -> Vec { + /// The amplitude sweep trusts the calibration, so each point commands the + /// very depth it expects to measure. + fn sweep_points(&self) -> Vec { let count = self.sweep_count.clamp(2, 64) as usize; let span = self.max_a - self.min_a; (0..count) - .map(|index| self.min_a + span * index as f64 / (count - 1) as f64) + .map(|index| { + let depth_a = self.min_a + span * index as f64 / (count - 1) as f64; + SweepPoint { + commanded_a: depth_a, + expected_a: depth_a, + } + }) .collect() } @@ -1303,44 +1586,67 @@ impl StageAA1Plugin { } /// Kick off the amplitude sweep: validate, then lease the modulation owner. - fn begin_sweep(&mut self, context: &mut PluginControlContext<'_>) { - if self.recording.is_active() || self.sweep.is_some() { - self.message = "A recording or sweep is already running".into(); + fn begin_sweep(&mut self, context: &mut impl RecordingControl) { + if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { + self.message = + "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); + return; + } + if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { + self.message = "Sweep needs max a > min a".into(); + return; + } + let points = self.sweep_points(); + let message = format!( + "Sweep: acquiring modulation lease for {} points…", + points.len() + ); + self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, message); + } + + /// Shared entry point for both leased recording runs (amplitude sweep and + /// single event-count point): validate the destination and the owner, then + /// acquire the modulation lease that holds the drive for the whole run. + fn begin_leased_sweep( + &mut self, + context: &mut impl RecordingControl, + kind: SweepKind, + points: Vec, + lock: Option, + message: String, + ) { + if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { + self.message = "A recording, sweep or a₀ lock is already running".into(); return; } if self.output_folder.trim().is_empty() { - self.message = "Set an output folder before sweeping".into(); + self.message = "Set an output folder before recording".into(); return; } if self.measurement_id.trim().is_empty() { - self.message = "Set a measurement id before sweeping".into(); + self.message = "Set a measurement id before recording".into(); return; } if !self.modulation_connected() { - self.message = "Modulation owner is not connected — cannot sweep".into(); + self.message = "Modulation owner is not connected — cannot drive the depth".into(); return; } - if self.min_a.partial_cmp(&0.0) != Some(std::cmp::Ordering::Greater) { - self.message = - "Set Sweep min a > 0 (a = 0 is the background reference, not a sweep point)".into(); - return; - } - if self.max_a.partial_cmp(&self.min_a) != Some(std::cmp::Ordering::Greater) { - self.message = "Sweep needs max a > min a".into(); + if points.is_empty() { + self.message = "Nothing to record: the run has no points".into(); return; } - let points = self.sweep_points(); let now_ms = now_unix_ms(); let lease_id = LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))); let ttl_ms = self.sweep_lease_ttl_ms(points.len()); let request = self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); let lease_req = request.request_id; - let _ = context.request_service(&request); - let total = points.len(); + context.request_service(&request); self.sweep = Some(Sweep { phase: SweepPhase::AcquiringLease, + kind, points, + lock, index: 0, lease_id, lease_granted: false, @@ -1353,11 +1659,42 @@ impl StageAA1Plugin { last_activity_ms: now_ms, stop_requested: false, }); - self.message = format!("Sweep: acquiring modulation lease for {total} points…"); + self.message = message; + } + + /// Record one atomic frequency point of the exact-event-count workflow. + /// + /// The armed lock's commanded depth is re-applied under a modulation lease — + /// which also locks the operator's drive settings out for the whole point, so + /// the amplitude provably cannot change during the recorded interval — and the + /// point is then recorded through the same coordinator as every other run. + fn begin_a0_point(&mut self, context: &mut impl RecordingControl) { + let Some(hz) = self.frequency_hz() else { + self.message = "No modulation frequency yet — arm the drive first".into(); + return; + }; + let Some(lock) = self.armed_lock().cloned() else { + self.message = format!( + "No converged a₀ lock for {} — press Find a₀ at this frequency first", + frequency_label(hz) + ); + return; + }; + let points = vec![SweepPoint { + commanded_a: lock.commanded_a, + expected_a: lock.target_a, + }]; + let message = format!( + "Event-count point at {}: leasing the drive at commanded a = {:.3} (a₀ = {:.3})…", + frequency_label(lock.frequency_hz), + lock.commanded_a, + lock.target_a + ); + self.begin_leased_sweep(context, SweepKind::EventCount, points, Some(lock), message); } /// Release the modulation lease (if held) and clear the sweep. - fn finish_sweep(&mut self, context: &mut PluginControlContext<'_>, message: String) { + fn finish_sweep(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(sweep) = self.sweep.take() { if sweep.lease_granted { let request = self.modulation_request( @@ -1367,35 +1704,36 @@ impl StageAA1Plugin { }, &sweep.lease_id, ); - let _ = context.request_service(&request); + context.request_service(&request); } } self.message = message; } /// Renew the modulation lease and retarget the drive at the current point. - fn send_sweep_depth(&mut self, context: &mut PluginControlContext<'_>) { + fn send_sweep_depth(&mut self, context: &mut impl RecordingControl) { let Some(sweep) = self.sweep.as_ref() else { return; }; let lease_id = sweep.lease_id.clone(); let remaining = sweep.total().saturating_sub(sweep.index); + let commanded_a = sweep.commanded_a(); let target_a = sweep.target_a(); let index = sweep.index; let total = sweep.total(); let ttl_ms = self.sweep_lease_ttl_ms(remaining); let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); - let _ = context.request_service(&renew); + context.request_service(&renew); let depth = self.modulation_request( ModulationCommandV1::SetOpticalDepth { - depth_a_milli: (target_a * 1_000.0).round().clamp(0.0, u32::MAX as f64) as u32, + depth_a_milli: depth_a_milli(commanded_a), }, &lease_id, ); let depth_req = depth.request_id; - let _ = context.request_service(&depth); + context.request_service(&depth); let now_ms = now_unix_ms(); if let Some(sweep) = self.sweep.as_mut() { @@ -1406,29 +1744,46 @@ impl StageAA1Plugin { sweep.point_started = false; sweep.last_activity_ms = now_ms; } - self.message = format!( - "Sweep point {}/{}: retargeting drive to a = {:.3}…", - index + 1, - total, - target_a - ); + self.message = if commanded_a == target_a { + format!( + "Sweep point {}/{total}: retargeting drive to a = {target_a:.3}…", + index + 1 + ) + } else { + format!( + "Event-count point: commanding a = {commanded_a:.3} for a measured a₀ = {target_a:.3}…" + ) + }; } /// Advance the amplitude sweep one control tick. Runs before /// `drive_recording`, so a point's recording starts on the same tick. - fn drive_sweep(&mut self, context: &mut PluginControlContext<'_>) { + fn drive_sweep(&mut self, context: &mut impl RecordingControl) { if self.sweep.is_none() { if std::mem::take(&mut self.sweep_pending) { self.begin_sweep(context); + } else if std::mem::take(&mut self.a0_point_pending) { + self.begin_a0_point(context); } return; } self.sweep_pending = false; + self.a0_point_pending = false; let now_ms = now_unix_ms(); - let (phase, stop_requested, lease_granted, depth_applied, last_activity_ms, index, total) = { + let ( + phase, + kind, + stop_requested, + lease_granted, + depth_applied, + last_activity_ms, + index, + total, + ) = { let sweep = self.sweep.as_ref().expect("sweep checked above"); ( sweep.phase, + sweep.kind, sweep.stop_requested, sweep.lease_granted, sweep.depth_applied, @@ -1483,9 +1838,16 @@ impl StageAA1Plugin { } SweepPhase::Settling => { let target = self.sweep.as_ref().map(Sweep::target_a).unwrap_or_default(); + // The amplitude sweep drives open-loop and accepts the coarse + // calibration band; an event-count point replays a depth that was + // already trimmed against `a₀`, so it holds the lock's band. + let tolerance = match kind { + SweepKind::Amplitude => sweep_tolerance(target), + SweepKind::EventCount => self.a0_tolerance.max(1e-3), + }; let settled = self .measured_a() - .is_some_and(|measured| (measured - target).abs() <= sweep_tolerance(target)); + .is_some_and(|measured| (measured - target).abs() <= tolerance); let dwell_ms = (self.settle_s.max(0.0) * 1_000.0) as u64; let mut start_recording = false; let mut settle_timed_out = false; @@ -1509,10 +1871,14 @@ impl StageAA1Plugin { } } if start_recording { - self.pending_role = Some(RecRole::Normal); + self.pending_role = Some(kind.role()); if settle_timed_out { + let measured = self + .measured_a() + .map_or_else(|| "—".into(), |value| format!("{value:.3}")); self.message = format!( - "Sweep point {}/{}: a did not settle at {target:.3} — recording anyway", + "Sweep point {}/{}: a did not settle at {target:.3} (measured {measured}) \ + — recording anyway", index + 1, total, ); @@ -1541,7 +1907,11 @@ impl StageAA1Plugin { let message = format!("Sweep aborted: {}", self.message); self.finish_sweep(context, message); } else if index + 1 >= total { - self.finish_sweep(context, format!("Sweep complete: {total} points recorded")); + let message = match kind { + SweepKind::Amplitude => format!("Sweep complete: {total} points recorded"), + SweepKind::EventCount => self.message.clone(), + }; + self.finish_sweep(context, message); } else { if let Some(sweep) = self.sweep.as_mut() { sweep.index += 1; @@ -1605,124 +1975,753 @@ impl StageAA1Plugin { } } - fn on_host_reply(&mut self, reply: &HostCommandReply) { - if reply.request_id == self.recording.cam_start_req { - match &reply.outcome { - HostCommandOutcome::RecordingStarted { - actual_raw_path, .. - } => { - self.recording.cam_raw_path = Some(actual_raw_path.clone()); - self.recording.last_activity_ms = now_unix_ms(); - } - HostCommandOutcome::Rejected { code, message } => { - // Stop the rest of the recording; drive_recording resolves the - // abort from the current phase on the next tick. - self.note(format!("Camera recording rejected ({code}): {message}")); - self.recording.cam_rejected = true; - self.recording.stop_requested = true; - } - _ => {} - } - } else if reply.request_id == self.recording.cam_stop_req { - match &reply.outcome { - HostCommandOutcome::RecordingFinalized { - actual_raw_path, .. - } => { - self.recording.cam_finalized_path = Some(actual_raw_path.clone()); - self.recording.cam_complete = true; - self.recording.last_activity_ms = now_unix_ms(); - } - HostCommandOutcome::RecordingPartial { - actual_raw_path, .. - } => { - self.recording.cam_finalized_path = Some(actual_raw_path.clone()); - self.recording.last_activity_ms = now_unix_ms(); - } - HostCommandOutcome::Rejected { code, message } => { - self.message = format!("Camera stop failed ({code}): {message}"); - self.recording.cam_rejected = true; - self.recording.last_activity_ms = now_unix_ms(); + // ---- exact event-count depth a₀ (ADR 013) ------------------------------ + + /// The lock stored for `hz`, whether or not it converged. + fn lock_for_frequency(&self, hz: f64) -> Option<&A0LockPoint> { + self.a0_locks + .iter() + .find(|lock| same_frequency(lock.frequency_hz, hz)) + } + + /// The lock that applies to the drive right now: same frequency, converged, + /// and aimed at the `a₀` currently entered. + fn armed_lock(&self) -> Option<&A0LockPoint> { + let hz = self.frequency_hz()?; + self.lock_for_frequency(hz) + .filter(|lock| lock.converged && (lock.target_a - self.a0_target).abs() <= 1e-6) + } + + fn a0_locks_path(&self) -> Option { + let folder = self.output_folder.trim(); + (!folder.is_empty()).then(|| Path::new(folder).join(A0_LOCK_FILE)) + } + + /// Store a finished lock, replacing any earlier one at the same frequency, + /// and mirror the table to disk. Returns a save failure for the caller to + /// append to its own message. + fn store_lock(&mut self, lock: A0LockPoint) -> Result<(), String> { + self.a0_locks + .retain(|existing| !same_frequency(existing.frequency_hz, lock.frequency_hz)); + self.a0_locks.push(lock); + self.a0_locks + .sort_by(|left, right| left.frequency_hz.total_cmp(&right.frequency_hz)); + self.save_a0_locks() + } + + /// Persist the lock table next to the recordings, so the found depths survive + /// a restart and can be cited offline. + /// + /// Returns the failure so the caller can append it to its own message: a + /// lock the operator can see on screen but that never reached disk is a + /// lock they will not have after a restart. + fn save_a0_locks(&mut self) -> Result<(), String> { + let Some(path) = self.a0_locks_path() else { + return Ok(()); + }; + let table = A0LockTable { + locks: self.a0_locks.clone(), + }; + let written = serde_json::to_string_pretty(&table) + .map_err(|error| error.to_string()) + .and_then(|text| { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; } - _ => {} - } - } + std::fs::write(&path, text).map_err(|error| error.to_string()) + }); + written.map_err(|error| format!("a₀ lock table save failed: {error}")) } - fn on_service_reply(&mut self, reply: &PluginServiceReply) { - if self.on_sweep_reply(reply) { + /// Re-read the lock table when the experiment folder changes. + fn load_a0_locks(&mut self) { + let folder = self.output_folder.trim().to_string(); + if self.loaded_locks_folder.as_deref() == Some(folder.as_str()) { return; } - let response = match &reply.outcome { - PluginServiceOutcome::Accepted { payload } => { - serde_json::from_value::(payload.clone()).ok() - } - PluginServiceOutcome::Rejected { code, message } => { - if reply.request_id == self.recording.connect_req - || reply.request_id == self.recording.lease_req - || reply.request_id == self.recording.pd_begin_req - { - self.note(format!("Photodiode start failed ({code}): {message}")); - self.recording.pd_rejected = true; - self.recording.stop_requested = true; - } else if reply.request_id == self.recording.pd_finalize_req { - self.note(format!("Photodiode save failed ({code}): {message}")); - self.recording.pd_rejected = true; - self.recording.lease_granted = false; - self.recording.last_activity_ms = now_unix_ms(); - } - None - } - }; - let Some(response) = response else { + self.loaded_locks_folder = Some(folder); + self.a0_locks.clear(); + let Some(path) = self.a0_locks_path() else { return; }; - if reply.request_id == self.recording.connect_req { - self.recording.connect_accepted = true; - self.recording.last_activity_ms = now_unix_ms(); - } else if reply.request_id == self.recording.lease_req { - self.recording.lease_granted = true; - self.recording.last_activity_ms = now_unix_ms(); - } else if reply.request_id == self.recording.pd_begin_req { - if let Some(PdqReceiptV1::Started(started)) = &response.receipt { - self.recording.pd_pdq_path = Some(started.pdq_path.clone()); - self.recording.pd_sidecar_path = Some(started.sidecar_path.clone()); - self.recording.last_activity_ms = now_unix_ms(); - } - } else if reply.request_id == self.recording.pd_finalize_req { - self.recording.pd_finalized = true; - if let Some(PdqReceiptV1::Finalized(finalized)) = &response.receipt { - self.recording.pd_pdq_path = Some(finalized.pdq_path.clone()); - self.recording.pd_sidecar_path = Some(finalized.sidecar_path.clone()); - self.recording.pd_valid = finalized.valid; - } - self.recording.lease_granted = false; - self.recording.last_activity_ms = now_unix_ms(); + if let Some(table) = std::fs::read_to_string(&path) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + { + self.a0_locks = table.locks; } } - /// Advance the recording state machine one control tick. - fn drive_recording(&mut self, context: &mut impl RecordingControl) { + /// Worst-case lock duration, used as the modulation lease TTL. + fn a0_lock_lease_ttl_ms(&self) -> u64 { + let per_trial_ms = (self.settle_s.max(0.0) * 1_000.0) as u64 + SWEEP_SETTLE_TIMEOUT_MS; + u64::from(A0_LOCK_MAX_TRIALS) + .saturating_mul(per_trial_ms) + .saturating_add(60_000) + } + + /// Kick off the closed-loop `a₀` lock at the current frequency. + fn begin_a0_lock(&mut self, context: &mut impl RecordingControl) { + if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { + self.message = "A recording, sweep or a₀ lock is already running".into(); + return; + } + if !self.modulation_connected() { + self.message = "Modulation owner is not connected — cannot find a₀".into(); + return; + } + // The lock table belongs to the experiment folder, and it is re-read + // whenever that folder changes: without one, a lock found now would be + // dropped the moment the operator picks the destination. + if self.output_folder.trim().is_empty() { + self.message = "Set an output folder before finding a₀".into(); + return; + } + let Some(hz) = self.frequency_hz() else { + self.message = "No modulation frequency yet — arm the drive before finding a₀".into(); + return; + }; + if self.measured_a().is_none() { + self.message = + "No photodiode-measured a — connect the photodiode and anchor I_tot first".into(); + return; + } + // Refuse before touching the drive, not after eight trials of chasing a + // truncated estimate upwards. + if let Err(reason) = self.optical_window_covers_a_cycle(hz) { + self.message = format!("Cannot find a₀ at {}: {reason}", frequency_label(hz)); + return; + } + let target = self.a0_target; + if !(COMMANDED_A_MIN..=COMMANDED_A_MAX).contains(&target) { + self.message = format!( + "a₀ = {target:.3} is outside the drivable {COMMANDED_A_MIN}..={COMMANDED_A_MAX}" + ); + return; + } + // Warm start from an earlier lock at this frequency; otherwise trust the + // Pockels calibration for the first guess (command exactly `a₀`). + let start = self + .lock_for_frequency(hz) + .map(|lock| lock.commanded_a) + .unwrap_or(target); let now_ms = now_unix_ms(); - match self.recording.phase { - RecPhase::Idle => { - if let Some(role) = self.pending_role.take() { - self.begin_recording(context, role); - } - } - RecPhase::StartingCamera => { - if self.recording.cam_rejected { - let message = self.message.clone(); - self.release_and_idle(context, message); - } else if self.recording.cam_raw_path.is_some() { - if self.recording.stop_requested { - self.stop_camera(context); - } else { - self.connect_photodiode(context); - } - } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS - { - self.recording.cam_rejected = true; - self.release_and_idle(context, "Timed out starting camera recording".into()); + let lease_id = LeaseId::new(format!("a1-a0-{}", format_compact_utc(now_ms / 1_000))); + let ttl_ms = self.a0_lock_lease_ttl_ms(); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + context.request_service(&request); + self.a0_lock = Some(A0Lock { + phase: A0LockPhase::AcquiringLease, + target_a: target, + tolerance: self.a0_tolerance.max(1e-3), + commanded_a: clamp_commanded_a(start), + frequency_hz: hz, + trial: 1, + samples: Vec::new(), + sampled_revision: None, + measure_from_ms: 0, + window_ms: 0, + deadline_ms: 0, + lease_id, + lease_granted: false, + lease_req, + depth_req: 0, + depth_applied: false, + last_activity_ms: now_ms, + stop_requested: false, + }); + self.message = format!( + "a₀ lock at {}: acquiring the modulation lease…", + frequency_label(hz) + ); + } + + /// Renew the lease and command the current trial's depth. + fn send_a0_depth(&mut self, context: &mut impl RecordingControl) { + let Some(lock) = self.a0_lock.as_ref() else { + return; + }; + let lease_id = lock.lease_id.clone(); + let commanded = lock.commanded_a; + let trial = lock.trial; + let target = lock.target_a; + + let ttl_ms = self.a0_lock_lease_ttl_ms(); + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + context.request_service(&renew); + let depth = self.modulation_request( + ModulationCommandV1::SetOpticalDepth { + depth_a_milli: depth_a_milli(commanded), + }, + &lease_id, + ); + let depth_req = depth.request_id; + context.request_service(&depth); + + let now_ms = now_unix_ms(); + if let Some(lock) = self.a0_lock.as_mut() { + lock.phase = A0LockPhase::SettingDepth; + lock.depth_req = depth_req; + lock.depth_applied = false; + lock.samples.clear(); + lock.sampled_revision = None; + lock.last_activity_ms = now_ms; + } + self.message = format!( + "a₀ lock trial {trial}/{A0_LOCK_MAX_TRIALS}: commanding a = {commanded:.3} for a \ + measured a₀ = {target:.3}…" + ); + } + + /// Release the modulation lease and clear the lock. + /// + /// Never `safe_off`: the drive must stay exactly where the lock left it, so + /// the event-count point that follows records at `a₀`. + fn finish_a0_lock(&mut self, context: &mut impl RecordingControl, message: String) { + if let Some(lock) = self.a0_lock.take() { + if lock.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 a0 lock finished".into(), + }, + &lock.lease_id, + ); + context.request_service(&request); + } + } + self.message = message; + } + + /// Length of the photodiode's contrast estimator window, in milliseconds. + /// + /// This is the time a commanded depth needs to fully replace the previous + /// one inside the estimate. Owners that predate the field do not publish + /// it; then only the operator's settle dwell is available. + fn optical_window_seconds(&self) -> Option { + self.photodiode + .as_ref()? + .optical_summary + .as_ref()? + .window_seconds + .filter(|seconds| seconds.is_finite() && *seconds > 0.0) + } + + /// [`Self::optical_window_seconds`] rounded up to the millisecond the lock's + /// timers work in. + fn optical_window_ms(&self) -> Option { + self.optical_window_seconds() + .map(|seconds| (seconds * 1_000.0).ceil() as u64) + } + + /// Whether the photodiode's estimator window spans at least one full + /// modulation cycle at `hz`, i.e. whether the published `a` can be a + /// peak-to-peak measurement at all. + /// + /// The owner refuses on its own when its markers can prove the window is + /// too short. It cannot when it has no marker stream — but A1 always knows + /// the frequency, from its own phase-0 triggers or the armed drive, so the + /// check is repeated here where the knowledge is. Getting this wrong is not + /// a small error: a sub-cycle window *under*-reports `a`, and the lock + /// divides by it, so it would drive the depth up until it rails. + fn optical_window_covers_a_cycle(&self, hz: f64) -> Result<(), String> { + let Some(window_seconds) = self.optical_window_seconds() else { + return Ok(()); + }; + let cycles = window_seconds * hz; + if cycles >= 1.0 { + return Ok(()); + } + Err(format!( + "the photodiode estimates a over {window_seconds:.4} s, only {cycles:.2} cycles at \ + {} — a is a peak-to-peak quantity and would be under-reported. Raise the photodiode \ + cache length to at least {:.0} s", + frequency_label(hz), + (2.0 / hz).ceil().max(1.0), + )) + } + + /// Take one reading per *independent* photodiode window. + /// + /// Two constraints, both about the estimator window rather than the + /// publisher: a reading must come from a summary that did not exist when + /// the depth was commanded (`sampled_revision`), and consecutive readings + /// must be at least [`A0_LOCK_SAMPLE_SPACING`] of a window apart — + /// otherwise they share nearly all their samples and three of them say no + /// more than one. + fn sample_a0_measurement(&mut self, now_ms: u64) { + let Some((revision, measured)) = self.photodiode.as_ref().and_then(|summary| { + summary + .optical_summary + .as_ref() + .map(|optical| (summary.service_revision, optical.measured_log_contrast)) + }) else { + return; + }; + let spacing_ms = self.a0_sample_spacing_ms(); + let Some(lock) = self.a0_lock.as_mut() else { + return; + }; + if now_ms < lock.measure_from_ms || lock.sampled_revision == Some(revision) { + return; + } + lock.sampled_revision = Some(revision); + lock.samples.push(measured); + lock.measure_from_ms = now_ms.saturating_add(spacing_ms); + } + + /// Minimum gap between two readings of one trial. + fn a0_sample_spacing_ms(&self) -> u64 { + let window_ms = self + .a0_lock + .as_ref() + .map(|lock| lock.window_ms) + .unwrap_or_default(); + ((window_ms as f64) * A0_LOCK_SAMPLE_SPACING).ceil() as u64 + } + + /// Photodiode clipping note for a lock message, empty when the windows are clean. + fn clip_warning(&self) -> String { + let Some(optical) = self + .photodiode + .as_ref() + .and_then(|summary| summary.optical_summary.as_ref()) + else { + return String::new(); + }; + if optical.low_clip_fraction.max(optical.high_clip_fraction) <= A0_LOCK_CLIP_WARNING { + return String::new(); + } + format!( + " — warning: photodiode clipping (low {:.1} %, high {:.1} %), the measured a is a \ + truncated estimate", + optical.low_clip_fraction * 100.0, + optical.high_clip_fraction * 100.0 + ) + } + + /// Close out one trial: converged, out of trials, at a drive limit, or one + /// more multiplicative correction. + fn evaluate_a0_trial(&mut self, context: &mut impl RecordingControl) { + let Some(lock) = self.a0_lock.as_ref() else { + return; + }; + let (target, tolerance, commanded, trial, hz) = ( + lock.target_a, + lock.tolerance, + lock.commanded_a, + lock.trial, + lock.frequency_hz, + ); + let mut readings = lock.samples.clone(); + if readings.is_empty() { + // The owner withholds `a` for a stated reason (clipping, no + // headroom, a bad `I_tot` anchor, a sub-cycle window). It does not + // publish the reason on the contract, so name the likely ones + // rather than leave the operator with "nothing happened". + self.finish_a0_lock( + context, + "a₀ lock aborted: the photodiode published no a while measuring — it withholds \ + one when the window clips, has no headroom above dark, the I_tot anchor is \ + below the signal, or the window is shorter than one modulation cycle" + .into(), + ); + return; + } + readings.sort_by(f64::total_cmp); + let measured = readings[readings.len() / 2]; + let spread = readings[readings.len() - 1] - readings[0]; + if measured <= 0.0 { + self.finish_a0_lock( + context, + format!( + "a₀ lock aborted: the photodiode measured a = {measured:.3} — check the I_tot \ + anchor and that the drive is modulating" + ), + ); + return; + } + // A drifting `a` that happens to cross the target on one reading is not + // a lock: the next action would record at whatever it drifted to. + if readings.len() > 1 && spread > tolerance * A0_LOCK_MAX_SPREAD_TOLERANCES { + self.finish_a0_lock( + context, + format!( + "a₀ lock aborted at {}: the measured a is not settled — {} readings spread \ + {spread:.3} across {}× the ±{tolerance:.3} tolerance (median {measured:.3}). \ + Increase Sweep settle (s) or check the drive and the I_tot anchor", + frequency_label(hz), + readings.len(), + A0_LOCK_MAX_SPREAD_TOLERANCES, + ), + ); + return; + } + + let converged = (measured - target).abs() <= tolerance; + // The delivered optical depth is proportional to the commanded one to + // first order, so one gain correction per trial converges in a couple of + // steps even where the drive rolls off at high frequency. + let ratio = (target / measured).clamp(1.0 / A0_LOCK_MAX_STEP_RATIO, A0_LOCK_MAX_STEP_RATIO); + let next = clamp_commanded_a(commanded * ratio); + let railed = !converged && (next - commanded).abs() < 1e-9; + let exhausted = trial >= A0_LOCK_MAX_TRIALS; + + if !converged && !railed && !exhausted { + if let Some(lock) = self.a0_lock.as_mut() { + lock.commanded_a = next; + lock.trial += 1; + } + self.message = format!( + "a₀ lock trial {trial}: measured a = {measured:.3} vs a₀ = {target:.3} — \ + correcting the commanded depth to {next:.3}" + ); + self.send_a0_depth(context); + return; + } + + let optical = self + .photodiode + .as_ref() + .and_then(|summary| summary.optical_summary.as_ref()); + let saved = self.store_lock(A0LockPoint { + frequency_hz: hz, + target_a: target, + commanded_a: commanded, + measured_a: measured, + trials: trial, + converged, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: optical.map(|optical| optical.low_clip_fraction), + high_clip_fraction: optical.map(|optical| optical.high_clip_fraction), + }); + let label = frequency_label(hz); + let message = if converged { + format!( + "a₀ locked at {label}: commanded a = {commanded:.3} measures a = {measured:.3} \ + (a₀ = {target:.3}, {trial} trial(s)){}", + self.clip_warning() + ) + } else if railed { + format!( + "a₀ lock stopped at {label}: commanded a = {commanded:.3} is at the drivable limit \ + and only measures a = {measured:.3} — lower a₀ or the operating point I_k" + ) + } else { + format!( + "a₀ lock did not converge at {label}: best commanded a = {commanded:.3} measures \ + a = {measured:.3} after {trial} trials — widen the tolerance or check the drive" + ) + }; + // A lock the operator can see but that never reached disk is a lock + // they will not have after a restart — say so on the same line. + let message = match saved { + Ok(()) => message, + Err(error) => format!("{message} — {error}"), + }; + self.finish_a0_lock(context, message); + } + + /// Advance the `a₀` lock one control tick. + fn drive_a0_lock(&mut self, context: &mut impl RecordingControl) { + if self.a0_lock.is_none() { + if std::mem::take(&mut self.a0_lock_pending) { + self.begin_a0_lock(context); + } + return; + } + self.a0_lock_pending = false; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, depth_applied, last_activity_ms) = { + let lock = self.a0_lock.as_ref().expect("lock checked above"); + ( + lock.phase, + lock.stop_requested, + lock.lease_granted, + lock.depth_applied, + lock.last_activity_ms, + ) + }; + if stop_requested { + let message = if self.message.is_empty() { + "a₀ lock stopped".into() + } else { + self.message.clone() + }; + self.finish_a0_lock(context, message); + return; + } + match phase { + A0LockPhase::AcquiringLease => { + if lease_granted { + self.send_a0_depth(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_a0_lock( + context, + "a₀ lock aborted: timed out acquiring the modulation lease".into(), + ); + } + } + A0LockPhase::SettingDepth => { + if depth_applied { + // The drive settles for the operator's dwell, and the + // photodiode's own estimator window has to roll over before + // the published `a` is free of the previous depth. Waiting + // for only the shorter of the two silently measures a + // mixture — with the 0.82 s default window that is every + // settle below ~1 s, and it gets worse at low frequency + // where the window grows to cover whole cycles. + let window_ms = self.optical_window_ms().unwrap_or_default(); + let dwell_ms = ((self.settle_s.max(0.0) * 1_000.0) as u64).max(window_ms); + // Only summaries published *after* this depth was commanded + // count, so the trial never averages the previous depth. + let published = self + .photodiode + .as_ref() + .map(|summary| summary.service_revision); + if let Some(lock) = self.a0_lock.as_mut() { + lock.phase = A0LockPhase::Measuring; + lock.window_ms = window_ms; + lock.measure_from_ms = now_ms.saturating_add(dwell_ms); + // The deadline has to outlast the readings it is + // waiting for, or a low-frequency point times out + // before its first independent sample can exist. + let sampling_ms = + (window_ms as f64 * A0_LOCK_SAMPLE_SPACING * A0_LOCK_SAMPLES as f64) + .ceil() as u64; + lock.deadline_ms = lock + .measure_from_ms + .saturating_add(SWEEP_SETTLE_TIMEOUT_MS.max(sampling_ms * 2)); + lock.samples.clear(); + lock.sampled_revision = published; + } + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_a0_lock( + context, + "a₀ lock aborted: timed out retargeting the modulation drive".into(), + ); + } + } + A0LockPhase::Measuring => { + self.sample_a0_measurement(now_ms); + let ready = self.a0_lock.as_ref().is_some_and(|lock| { + lock.samples.len() >= A0_LOCK_SAMPLES || now_ms >= lock.deadline_ms + }); + if ready { + self.evaluate_a0_trial(context); + } + } + } + } + + /// Routes modulation-service replies belonging to the `a₀` lock. Returns true + /// when the reply was consumed. + fn on_a0_lock_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some((lease_req, depth_req)) = self + .a0_lock + .as_ref() + .map(|lock| (lock.lease_req, lock.depth_req)) + else { + return false; + }; + let abort = |this: &mut Self, message: String| { + this.message = message; + if let Some(lock) = this.a0_lock.as_mut() { + lock.stop_requested = true; + } + }; + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(lock) = self.a0_lock.as_mut() { + lock.lease_granted = true; + lock.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("a₀ lock aborted: modulation lease rejected: {message}"), + ), + } + true + } else if reply.request_id == depth_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(lock) = self.a0_lock.as_mut() { + lock.depth_applied = true; + lock.last_activity_ms = now_unix_ms(); + } + } + // The owner refuses a depth its calibrated drive cannot express + // (lobe ceiling, DAC limit) — that *is* the "a₀ unreachable at + // this operating point" answer, so surface its wording verbatim. + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("a₀ lock aborted: the drive rejected the commanded depth: {message}"), + ), + } + true + } else { + false + } + } + + fn a0_locks_dataset(&self) -> TableDatasetV1 { + let column = |id: &str, values: Vec| TableColumnData { + column_id: id.into(), + values: TableColumnValues::String(values), + }; + let map = |select: fn(&A0LockPoint) -> String| { + self.a0_locks.iter().map(select).collect::>() + }; + TableDatasetV1 { + columns: vec![ + column("frequency", map(|lock| frequency_label(lock.frequency_hz))), + column("target_a", map(|lock| format!("{:.3}", lock.target_a))), + column( + "commanded_a", + map(|lock| format!("{:.3}", lock.commanded_a)), + ), + column("measured_a", map(|lock| format!("{:.3}", lock.measured_a))), + column("trials", map(|lock| lock.trials.to_string())), + column( + "state", + map(|lock| { + if lock.converged { + "locked".into() + } else { + "not converged".into() + } + }), + ), + column( + "locked_at", + map(|lock| format_iso_utc(lock.locked_at_unix_ms / 1_000)), + ), + ], + } + } + + fn on_host_reply(&mut self, reply: &HostCommandReply) { + if reply.request_id == self.recording.cam_start_req { + match &reply.outcome { + HostCommandOutcome::RecordingStarted { + actual_raw_path, .. + } => { + self.recording.cam_raw_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + // Stop the rest of the recording; drive_recording resolves the + // abort from the current phase on the next tick. + self.note(format!("Camera recording rejected ({code}): {message}")); + self.recording.cam_rejected = true; + self.recording.stop_requested = true; + } + _ => {} + } + } else if reply.request_id == self.recording.cam_stop_req { + match &reply.outcome { + HostCommandOutcome::RecordingFinalized { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.cam_complete = true; + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::RecordingPartial { + actual_raw_path, .. + } => { + self.recording.cam_finalized_path = Some(actual_raw_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + HostCommandOutcome::Rejected { code, message } => { + self.message = format!("Camera stop failed ({code}): {message}"); + self.recording.cam_rejected = true; + self.recording.last_activity_ms = now_unix_ms(); + } + _ => {} + } + } + } + + fn on_service_reply(&mut self, reply: &PluginServiceReply) { + if self.on_sweep_reply(reply) || self.on_a0_lock_reply(reply) { + return; + } + let response = match &reply.outcome { + PluginServiceOutcome::Accepted { payload } => { + serde_json::from_value::(payload.clone()).ok() + } + PluginServiceOutcome::Rejected { code, message } => { + if reply.request_id == self.recording.connect_req + || reply.request_id == self.recording.lease_req + || reply.request_id == self.recording.pd_begin_req + { + self.note(format!("Photodiode start failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.stop_requested = true; + } else if reply.request_id == self.recording.pd_finalize_req { + self.note(format!("Photodiode save failed ({code}): {message}")); + self.recording.pd_rejected = true; + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + None + } + }; + let Some(response) = response else { + return; + }; + if reply.request_id == self.recording.connect_req { + self.recording.connect_accepted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.lease_req { + self.recording.lease_granted = true; + self.recording.last_activity_ms = now_unix_ms(); + } else if reply.request_id == self.recording.pd_begin_req { + if let Some(PdqReceiptV1::Started(started)) = &response.receipt { + self.recording.pd_pdq_path = Some(started.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(started.sidecar_path.clone()); + self.recording.last_activity_ms = now_unix_ms(); + } + } else if reply.request_id == self.recording.pd_finalize_req { + self.recording.pd_finalized = true; + if let Some(PdqReceiptV1::Finalized(finalized)) = &response.receipt { + self.recording.pd_pdq_path = Some(finalized.pdq_path.clone()); + self.recording.pd_sidecar_path = Some(finalized.sidecar_path.clone()); + self.recording.pd_valid = finalized.valid; + } + self.recording.lease_granted = false; + self.recording.last_activity_ms = now_unix_ms(); + } + } + + /// Advance the recording state machine one control tick. + fn drive_recording(&mut self, context: &mut impl RecordingControl) { + let now_ms = now_unix_ms(); + match self.recording.phase { + RecPhase::Idle => { + if let Some(role) = self.pending_role.take() { + self.begin_recording(context, role); + } + } + RecPhase::StartingCamera => { + if self.recording.cam_rejected { + let message = self.message.clone(); + self.release_and_idle(context, message); + } else if self.recording.cam_raw_path.is_some() { + if self.recording.stop_requested { + self.stop_camera(context); + } else { + self.connect_photodiode(context); + } + } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS + { + self.recording.cam_rejected = true; + self.release_and_idle(context, "Timed out starting camera recording".into()); } } RecPhase::ConnectingPhotodiode => { @@ -1853,10 +2852,24 @@ impl StageAA1Plugin { min_a: self.min_a, max_a: self.max_a, requested_a: point.map(Sweep::target_a), + commanded_a: point.map(Sweep::commanded_a), point_index: point.map(|sweep| sweep.index + 1), point_total: point.map(Sweep::total), } }, + a0_lock: self + .sweep + .as_ref() + .and_then(|sweep| sweep.lock.as_ref()) + .map(|lock| A0LockSidecar { + target_a: lock.target_a, + commanded_a: lock.commanded_a, + measured_a_at_lock: lock.measured_a, + frequency_hz_at_lock: lock.frequency_hz, + trials: lock.trials, + converged: lock.converged, + locked_at_utc: format_iso_utc(lock.locked_at_unix_ms / 1_000), + }), pilot: (self.recording.role == RecRole::Pilot) .then_some(self.pilot_windows) .flatten() @@ -1926,6 +2939,9 @@ struct SidecarDoc { finalized_at_utc: String, duration_s: u64, sweep: SweepSidecar, + /// Present on **event-count** points: the `a₀` lock this point replayed. + #[serde(skip_serializing_if = "Option::is_none")] + a0_lock: Option, #[serde(skip_serializing_if = "Option::is_none")] pilot: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1945,6 +2961,11 @@ struct SweepSidecar { /// `[optical]`); absent on manual recordings. #[serde(skip_serializing_if = "Option::is_none")] requested_a: Option, + /// The depth the drive was *commanded* to for this point. Equal to + /// `requested_a` on the amplitude sweep; on an event-count point it is the + /// `a₀`-locked depth, which differs by the drive roll-off at that frequency. + #[serde(skip_serializing_if = "Option::is_none")] + commanded_a: Option, /// 1-based point position within the sweep; absent on manual recordings. #[serde(skip_serializing_if = "Option::is_none")] point_index: Option, @@ -1952,6 +2973,19 @@ struct SweepSidecar { point_total: Option, } +/// The `a₀` lock an **event-count** point replayed: the closed-loop trim that +/// made the photodiode measure the frozen `a₀` at this frequency. +#[derive(Serialize)] +struct A0LockSidecar { + target_a: f64, + commanded_a: f64, + measured_a_at_lock: f64, + frequency_hz_at_lock: f64, + trials: u32, + converged: bool, + locked_at_utc: String, +} + /// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read /// back to reuse them across the row. #[derive(Serialize, serde::Deserialize)] @@ -2350,8 +3384,11 @@ impl Plugin for StageAA1Plugin { // the folder or id changes, look them up in the folder. if !self.recording.is_active() { self.scan_measurement_folder(); + self.load_a0_locks(); } - // The sweep runs first so a point's recording starts on the same tick. + // The lock and the sweep run first so a point's recording starts on the + // same tick. They are mutually exclusive, guarded when they begin. + self.drive_a0_lock(context); self.drive_sweep(context); self.drive_recording(context); // The fold reflects live snapshots (T, a) even between frames. @@ -2512,19 +3549,115 @@ impl Plugin for StageAA1Plugin { selected." .into(), ), - kind: SettingKind::Button { - enabled: can_record, + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_pilot".into(), + label: "Record pilot (freeze ON/OFF windows)".into(), + tooltip: Some( + "Records a bright reference for this row into the same folder \ + (…_pilot) and freezes the ON/OFF windows from the current live \ + signal. Set a high, non-saturating a in the modulation plugin \ + first. The frozen windows are reused for the whole row's q_p. \ + Disabled until an output folder is selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "record_background".into(), + label: "Record background (a≈0 floor)".into(), + tooltip: Some( + "Records an unmodulated reference (…_background) and captures the \ + false-response floor q0 in the current windows. Set a≈0 in the \ + modulation plugin first. Disabled until an output folder is \ + selected." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, + SettingItem { + key: "stop_recording".into(), + label: "Stop (abort recording / sweep)".into(), + tooltip: Some( + "Stop and finalize the current recording before the duration \ + ends; during a sweep this also aborts the remaining points." + .into(), + ), + kind: SettingKind::Button { enabled: true }, + }, + ], + }, + SettingsSection { + label: "Exact event-count depth a₀".into(), + description: Some( + "Second Stage-A workflow, on top of the minimum-depth sweep above: hold \ + ONE photodiode-measured depth a₀ = ln(I_exc,max / I_exc,min) constant \ + across the frequency sweep. Freeze the flux point, camera configuration \ + and references first (pilot and background are recorded above), then per \ + frequency: set f in the modulation plugin, press Find a₀ — A1 leases the \ + drive and trims the *commanded* depth until the photodiode *measures* a₀ \ + — and then press Record a₀ point, which re-applies that depth under the \ + same lease (so the amplitude cannot change during the recorded interval) \ + and records one atomic RAW + PDQ + sidecar point named …_ec_fHz. The \ + found depths are kept per frequency, listed in the a₀ lock table view and \ + mirrored to a0_locks.json in the output folder. Randomising the frequency \ + order, interleaving the low-frequency reference and repeating blocks stay \ + yours — every point is one button press." + .into(), + ), + default_open: false, + items: vec![ + SettingItem { + key: "a0_target".into(), + label: "a₀ (measured log contrast)".into(), + tooltip: Some( + "The one photodiode-measured depth held across the whole frequency \ + sweep — never a DAC excursion. Pick it from the low-frequency \ + scout: high enough for several events per pixel per half-cycle, \ + still proportional (not saturated), and refractory-safe at the \ + highest frequency." + .into(), + ), + kind: SettingKind::F64Drag { + min: COMMANDED_A_MIN, + max: COMMANDED_A_MAX, + speed: 0.01, + default: self.a0_target, + }, + }, + SettingItem { + key: "a0_tolerance".into(), + label: "a₀ tolerance (absolute)".into(), + tooltip: Some( + "Convergence band on |measured a − a₀| for the lock, and the \ + settle band an event-count point must hold before it records." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.002, + max: 0.5, + speed: 0.002, + default: self.a0_tolerance, }, }, SettingItem { - key: "record_pilot".into(), - label: "Record pilot (freeze ON/OFF windows)".into(), + key: "find_a0".into(), + label: "Find a₀ (lock the drive depth)".into(), tooltip: Some( - "Records a bright reference for this row into the same folder \ - (…_pilot) and freezes the ON/OFF windows from the current live \ - signal. Set a high, non-saturating a in the modulation plugin \ - first. The frozen windows are reused for the whole row's q_p. \ - Disabled until an output folder is selected." + "Leases the modulation owner and iterates commanded a ← commanded \ + a · a₀/measured a until the photodiode-measured depth is a₀ at the \ + current frequency (up to 8 trials, waiting Sweep settle (s) per \ + trial). Records nothing, leaves the drive at the depth it found, \ + and stores it for this frequency. Requires a calibrated \ + periodic/optical drive armed in the modulation plugin and a \ + photodiode-measured a. Disabled until an output folder is selected." .into(), ), kind: SettingKind::Button { @@ -2532,13 +3665,14 @@ impl Plugin for StageAA1Plugin { }, }, SettingItem { - key: "record_background".into(), - label: "Record background (a≈0 floor)".into(), + key: "record_a0_point".into(), + label: "Record a₀ point (event-count)".into(), tooltip: Some( - "Records an unmodulated reference (…_background) and captures the \ - false-response floor q0 in the current windows. Set a≈0 in the \ - modulation plugin first. Disabled until an output folder is \ - selected." + "Records one atomic frequency point at the locked depth: re-applies \ + the found commanded a under a modulation lease, waits for the \ + measured a to hold a₀, then records camera RAW + photodiode PDQ + \ + sidecar under one run id (…_ec_fHz). Needs a converged lock for \ + the current frequency and an output folder." .into(), ), kind: SettingKind::Button { @@ -2546,11 +3680,12 @@ impl Plugin for StageAA1Plugin { }, }, SettingItem { - key: "stop_recording".into(), - label: "Stop (abort recording / sweep)".into(), + key: "clear_a0_locks".into(), + label: "Clear a₀ lock table".into(), tooltip: Some( - "Stop and finalize the current recording before the duration \ - ends; during a sweep this also aborts the remaining points." + "Drops every stored per-frequency lock and rewrites \ + a0_locks.json. Use it after changing the flux point, the \ + calibration or a₀ itself." .into(), ), kind: SettingKind::Button { enabled: true }, @@ -2684,6 +3819,11 @@ impl Plugin for StageAA1Plugin { "clear" => Some(self.press_clear.value()), "record_point" => Some(self.press_record_point.value()), "clear_curve" => Some(self.press_clear_curve.value()), + "a0_target" => Some(json!(self.a0_target)), + "a0_tolerance" => Some(json!(self.a0_tolerance)), + "find_a0" => Some(self.press_find_a0.value()), + "record_a0_point" => Some(self.press_record_a0.value()), + "clear_a0_locks" => Some(self.press_clear_a0.value()), // New id regenerates the measurement id locally; the id itself is // what synchronizes, so the press must not be forwarded (both // instances would generate different ids). @@ -2768,7 +3908,13 @@ impl Plugin for StageAA1Plugin { sweep.stop_requested = true; self.message = "Sweep stop requested".into(); } + if let Some(lock) = self.a0_lock.as_mut() { + lock.stop_requested = true; + self.message = "a₀ lock stop requested".into(); + } self.sweep_pending = false; + self.a0_lock_pending = false; + self.a0_point_pending = false; } } "live" => { @@ -2808,6 +3954,37 @@ impl Plugin for StageAA1Plugin { self.response_points.clear(); } } + "a0_target" => { + self.a0_target = value + .as_f64() + .ok_or("a0_target must be a number")? + .clamp(COMMANDED_A_MIN, COMMANDED_A_MAX); + } + "a0_tolerance" => { + self.a0_tolerance = value + .as_f64() + .ok_or("a0_tolerance must be a number")? + .clamp(0.002, 0.5); + } + "find_a0" => { + if self.press_find_a0.accept(&value) { + self.a0_lock_pending = true; + } + } + "record_a0_point" => { + if self.press_record_a0.accept(&value) { + self.a0_point_pending = true; + } + } + "clear_a0_locks" => { + if self.press_clear_a0.accept(&value) { + self.a0_locks.clear(); + self.message = match self.save_a0_locks() { + Ok(()) => "a₀ lock table cleared".into(), + Err(error) => error, + }; + } + } "new_id" => return Ok(()), _ => return Err(format!("unknown setting '{key}'")), } @@ -2836,13 +4013,34 @@ impl Plugin for StageAA1Plugin { SweepPhase::Settling => "settling", SweepPhase::Recording => "recording", }; + let label = match sweep.kind { + SweepKind::Amplitude => "Sweep", + SweepKind::EventCount => "Event-count point", + }; entries.push(StatusEntry::Text(format!( - "Sweep: point {}/{} at a → {:.3} ({phase})", + "{label}: point {}/{} commanding a = {:.3} for a measured {:.3} ({phase})", sweep.index + 1, sweep.total(), + sweep.commanded_a(), sweep.target_a() ))); } + if let Some(lock) = &self.a0_lock { + let phase = match lock.phase { + A0LockPhase::AcquiringLease => "leasing modulation", + A0LockPhase::SettingDepth => "commanding depth", + A0LockPhase::Measuring => "measuring", + }; + entries.push(StatusEntry::Text(format!( + "a₀ lock at {}: trial {}/{A0_LOCK_MAX_TRIALS} commanding a = {:.3} for a₀ = {:.3} \ + ({phase}, {} sample(s))", + frequency_label(lock.frequency_hz), + lock.trial, + lock.commanded_a, + lock.target_a, + lock.samples.len() + ))); + } if !self.message.is_empty() { entries.push(StatusEntry::Text(self.message.clone())); } @@ -2914,6 +4112,21 @@ impl Plugin for StageAA1Plugin { "Background floor: q0_on = {q0_on:.3}, q0_off = {q0_off:.3}" ))); } + entries.push(StatusEntry::Text(match self.armed_lock() { + Some(lock) => format!( + "a₀ = {:.3} armed at {}: commanded a = {:.3} (measured {:.3}); {} lock(s) stored", + lock.target_a, + frequency_label(lock.frequency_hz), + lock.commanded_a, + lock.measured_a, + self.a0_locks.len() + ), + None => format!( + "a₀ = {:.3}: no lock for this frequency — press Find a₀; {} lock(s) stored", + self.a0_target, + self.a0_locks.len() + ), + })); entries } @@ -2964,6 +4177,25 @@ impl Plugin for StageAA1Plugin { display: None, relations: Vec::new(), }, + HostDatasetDescriptor { + id: A0_LOCK_DATASET_ID.into(), + title: "A1 a₀ locks — commanded depth per frequency".into(), + kind: HostDatasetKind::TableV1(TableSchema { + columns: vec![ + column("frequency", "Frequency"), + column("target_a", "a₀ (target)"), + column("commanded_a", "Commanded a"), + column("measured_a", "Measured a"), + column("trials", "Trials"), + column("state", "State"), + column("locked_at", "Locked at (UTC)"), + ], + ..TableSchema::default() + }), + empty_message: "No a₀ lock yet — set a₀ and press Find a₀ per frequency".into(), + display: None, + relations: Vec::new(), + }, ], views: vec![ HostViewDescriptor { @@ -2987,6 +4219,13 @@ impl Plugin for StageAA1Plugin { placement: HostViewPlacement::Window, kind: HostViewKind::LineSeriesWindow, }, + HostViewDescriptor { + id: A0_LOCK_VIEW_ID.into(), + title: "A1 a₀ locks (commanded depth per frequency)".into(), + dataset_id: A0_LOCK_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::TableWindow, + }, ], actions: Vec::new(), } @@ -2997,6 +4236,7 @@ impl Plugin for StageAA1Plugin { STATUS_DATASET_ID => serde_json::to_vec(&self.status_dataset()).ok(), ROLLING_DATASET_ID => serde_json::to_vec(&self.rolling_dataset()).ok(), RESPONSE_CURVE_DATASET_ID => serde_json::to_vec(&self.response_curve_dataset()).ok(), + A0_LOCK_DATASET_ID => serde_json::to_vec(&self.a0_locks_dataset()).ok(), _ => None, } } @@ -3004,7 +4244,7 @@ impl Plugin for StageAA1Plugin { fn host_view_dataset_generation(&self, dataset_id: &str) -> u64 { matches!( dataset_id, - STATUS_DATASET_ID | ROLLING_DATASET_ID | RESPONSE_CURVE_DATASET_ID + STATUS_DATASET_ID | ROLLING_DATASET_ID | RESPONSE_CURVE_DATASET_ID | A0_LOCK_DATASET_ID ) .then_some(self.dataset_generation) .unwrap_or(0) @@ -3047,6 +4287,7 @@ mod tests { } } + /// Mirrors the ordering of [`StageAA1Plugin::process_control`]. fn control_tick( plugin: &mut StageAA1Plugin, inbox: PluginControlInbox, @@ -3058,9 +4299,210 @@ mod tests { for reply in &inbox.service_replies { plugin.on_service_reply(reply); } + plugin.drive_a0_lock(sink); + plugin.drive_sweep(sink); plugin.drive_recording(sink); } + /// Bare `Accepted` reply, as the modulation owner answers a lease or depth + /// command (only the outcome variant is routed). + fn accepted(request_id: u64) -> PluginServiceReply { + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: MODULATION_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Accepted { + payload: Value::Null, + }, + } + } + + fn rejected(request_id: u64, message: &str) -> PluginServiceReply { + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: MODULATION_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_MODULATION_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Rejected { + code: "invalid_command".into(), + message: message.into(), + }, + } + } + + fn connected_modulation() -> ModulationStateV1 { + ModulationStateV1 { + contract_version: stage_a_plugin_contract::CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("mod-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: Some("0.4.0".into()), + }, + capabilities: Vec::new(), + lease: None, + controller_state: stage_a_plugin_contract::ControllerStateV1::Configured, + active_run_id: None, + requested: None, + acknowledged: None, + synchronization: stage_a_plugin_contract::SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: stage_a_plugin_contract::FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 5_000, + }, + calibration_id: Some("pockels-test".into()), + } + } + + /// A photodiode snapshot reporting `measured_a`, published at `revision`. + fn photodiode_measuring(revision: u64, measured_a: f64) -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: stage_a_plugin_contract::CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: revision, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: None, + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: stage_a_plugin_contract::PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: None, + integrity: StreamIntegrityV1::default(), + level: None, + }, + active_recording: None, + last_finalized_recording: None, + optical_summary: Some(stage_a_plugin_contract::PhotodiodeOpticalSummaryV1 { + run_id: RunId::new("pd-run"), + calibration: stage_a_plugin_contract::PhotodiodeCalibrationV1 { + adc_calibration_id: "adc".into(), + dark_id: "dark".into(), + anchor_id: "anchor".into(), + dark_volts: 0.0, + total_power_volts: 1.0, + }, + measured_log_contrast: measured_a, + log_contrast_stddev: None, + excitation_min_volts: 0.1, + excitation_max_volts: 0.9, + excitation_headroom_volts: 0.1, + low_clip_fraction: 0.0, + high_clip_fraction: 0.0, + measured_frequency_hz: None, + fundamental_phase_rad: None, + total_harmonic_distortion: None, + // A short window, so the lock's dwell and sample spacing stay + // in the millisecond range the tests tick at. + window_seconds: Some(0.001), + covered_cycles: Some(8.0), + }), + synchronization: stage_a_plugin_contract::SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: stage_a_plugin_contract::FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 5_000, + }, + } + } + + /// The depth carried by the newest `SetOpticalDepth` the plugin emitted. + fn last_commanded_depth(sink: &ControlSink) -> Option { + sink.services.iter().rev().find_map(|request| { + let envelope: ModulationRequestV1 = + serde_json::from_value(request.payload.clone()).ok()?; + match envelope.command { + ModulationCommandV1::SetOpticalDepth { depth_a_milli } => { + Some(f64::from(depth_a_milli) / 1_000.0) + } + _ => None, + } + }) + } + + /// A unique scratch directory for a test's lock table and sidecars. + fn temp_folder(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!("a1-{tag}-{}", now_unix_ms())) + } + + /// A plugin wired to a connected drive at 1 kHz (marker-anchored) whose + /// photodiode reports a bench that delivers `gain ×` the commanded depth. + fn plugin_locking(gain: f64, folder: &Path) -> StageAA1Plugin { + let mut plugin = plugin_with_markers(); + plugin.modulation = Some(connected_modulation()); + plugin.photodiode = Some(photodiode_measuring(1, gain)); + plugin.output_folder = folder.display().to_string(); + plugin.measurement_id = "A1-ec".into(); + plugin.settle_s = 0.0; + plugin.a0_tolerance = 0.02; + plugin + } + + /// Answers the lock's outstanding lease/depth request and publishes the + /// photodiode readings the commanded depth produces, until the lock ends. + fn run_lock_to_completion( + plugin: &mut StageAA1Plugin, + sink: &mut ControlSink, + gain: f64, + max_ticks: usize, + ) -> usize { + let mut revision = 1; + // The first tick consumes the latched press and starts the lock. + control_tick(plugin, PluginControlInbox::default(), sink); + for tick in 0..max_ticks { + if plugin.a0_lock.is_none() { + return tick + 1; + } + let (lease_req, depth_req, granted, applied) = { + let lock = plugin.a0_lock.as_ref().expect("lock"); + ( + lock.lease_req, + lock.depth_req, + lock.lease_granted, + lock.depth_applied, + ) + }; + let mut replies = Vec::new(); + if !granted { + replies.push(accepted(lease_req)); + } else if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + // Measuring: publish what the bench delivers for the commanded + // depth as a fresh summary. + let commanded = plugin.a0_lock.as_ref().expect("lock").commanded_a; + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, commanded * gain)); + // The lock spaces its readings by a fraction of the photodiode's + // estimator window (1 ms in these fixtures), so a tick loop that + // never advances the wall clock would collect exactly one. + std::thread::sleep(std::time::Duration::from_millis(1)); + } + control_tick( + plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + sink, + ); + } + max_ticks + } + fn pd_reply(request_id: u64, receipt: Option) -> PluginServiceReply { let response = PhotodiodeResponseV1 { common: ResponseCommonV1 { @@ -3276,8 +4718,14 @@ mod tests { // must not record those as if they had come from this run. let mut plugin = plugin_with_markers(); plugin.pilot_windows = Some(( - PhaseWindow { start: 0.0, end: 0.2 }, - PhaseWindow { start: 0.5, end: 0.7 }, + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, )); // No events => the fold carries no signal => the freeze cannot pick // windows and must not leave the loaded ones in place. @@ -3536,9 +4984,13 @@ mod tests { }; let points = plugin.sweep_points(); assert_eq!(points.len(), 5); - assert!((points[0] - 0.5).abs() < 1e-12); - assert!((points[4] - 2.5).abs() < 1e-12); - assert!((points[2] - 1.5).abs() < 1e-12); + assert!((points[0].expected_a - 0.5).abs() < 1e-12); + assert!((points[4].expected_a - 2.5).abs() < 1e-12); + assert!((points[2].expected_a - 1.5).abs() < 1e-12); + // The amplitude sweep trusts the calibration: it commands what it expects. + assert!(points + .iter() + .all(|point| point.commanded_a == point.expected_a)); } #[test] @@ -3546,9 +4998,12 @@ mod tests { let mut plugin = plugin_with_markers(); plugin.min_a = 0.5; plugin.max_a = 1.5; + plugin.sweep_count = 3; plugin.sweep = Some(Sweep { phase: SweepPhase::Recording, - points: vec![0.5, 1.0, 1.5], + kind: SweepKind::Amplitude, + points: plugin.sweep_points(), + lock: None, index: 1, lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, @@ -3574,6 +5029,360 @@ mod tests { let _ = std::fs::remove_file(&path); } + #[test] + fn a0_lock_trims_the_commanded_depth_until_the_photodiode_measures_a0() { + // A bench that delivers 60 % of the commanded depth (drive roll-off): + // commanding a₀ directly would record a = 0.30 instead of 0.50. + let folder = temp_folder("lock"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + + let ticks = run_lock_to_completion(&mut plugin, &mut sink, 0.6, 64); + assert!(ticks < 64, "lock never finished"); + + let lock = plugin + .a0_locks + .first() + .expect("the converged lock is stored"); + assert!(lock.converged, "message: {}", plugin.message); + assert!( + (lock.measured_a - 0.5).abs() <= plugin.a0_tolerance, + "measured {}", + lock.measured_a + ); + assert!( + (lock.commanded_a - 0.5 / 0.6).abs() < 0.01, + "commanded {}", + lock.commanded_a + ); + assert!(lock.trials >= 2, "trials {}", lock.trials); + assert!((lock.frequency_hz - 1_000.0).abs() < 1.0); + // The drive is left at the depth the lock found, and the lease is + // released without a safe-off so it stays there for the recording. + assert!((last_commanded_depth(&sink).expect("depth") - lock.commanded_a).abs() < 0.002); + let release: ModulationRequestV1 = + serde_json::from_value(sink.services.last().expect("release").payload.clone()) + .expect("envelope"); + assert!(matches!( + release.command, + ModulationCommandV1::ReleaseLease { + safe_off: false, + .. + } + )); + // The lock arms the event-count recording for this frequency. + assert!(plugin.armed_lock().is_some()); + // …and the table is on disk next to the recordings. + assert!(folder.join(A0_LOCK_FILE).exists()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_refuses_a_photodiode_window_shorter_than_one_cycle() { + // `a` is peak-to-peak. Under one cycle the photodiode under-reports it, + // and the lock divides by it — so it would inflate the drive until it + // railed. Refuse before touching the drive, and say what to change. + let folder = temp_folder("subcycle"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + // 1 kHz markers give the plugin its frequency; make the estimator + // window 0.4 ms, i.e. 0.4 of a cycle. + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(0.000_4); + } + } + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!(plugin.a0_lock.is_none(), "the lock must not start"); + assert!(sink.services.is_empty(), "no lease may be requested"); + assert!( + plugin.message.contains("0.40 cycles") && plugin.message.contains("cache length"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_refuses_to_lock_onto_an_unsettled_operating_point() { + // Readings that walk across the target are not a lock: the next action + // would record at wherever the drive drifted to, not at a₀. + let folder = temp_folder("unsettled"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + let mut revision = 1; + let mut drift = 0.30; + for _ in 0..64 { + if plugin.a0_lock.is_none() { + break; + } + let (lease_req, depth_req, granted, applied) = { + let lock = plugin.a0_lock.as_ref().expect("lock"); + ( + lock.lease_req, + lock.depth_req, + lock.lease_granted, + lock.depth_applied, + ) + }; + let mut replies = Vec::new(); + if !granted { + replies.push(accepted(lease_req)); + } else if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + drift += 0.20; // 0.50, 0.70, 0.90 — straddling a₀ = 0.50 + plugin.photodiode = Some(photodiode_measuring(revision, drift)); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); + } + + assert!(plugin.a0_lock.is_none(), "the lock must end"); + assert!( + plugin.message.contains("not settled"), + "message: {}", + plugin.message + ); + // Nothing is stored, so nothing can arm a recording. + assert!(plugin.a0_locks.is_empty()); + assert!(plugin.armed_lock().is_none()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_reports_an_unreachable_depth_instead_of_arming_a_recording() { + // The bench delivers 5 % of the commanded depth: a₀ = 0.5 would need a + // commanded depth far beyond what the owner accepts. + let folder = temp_folder("unreachable"); + let mut plugin = plugin_locking(0.05, &folder); + plugin.a0_target = 0.5; + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + + assert!(run_lock_to_completion(&mut plugin, &mut sink, 0.05, 256) < 256); + let lock = plugin.a0_locks.first().expect("the attempt is recorded"); + assert!(!lock.converged); + assert!((lock.commanded_a - COMMANDED_A_MAX).abs() < 1e-9); + assert!( + plugin.message.contains("drivable limit") + || plugin.message.contains("did not converge"), + "message: {}", + plugin.message + ); + // A non-converged lock must never arm an event-count recording. + assert!(plugin.armed_lock().is_none()); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a0_lock_surfaces_a_drive_rejection_verbatim() { + let folder = temp_folder("reject"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_lock_pending = true; + let mut sink = ControlSink::default(); + // Tick 1: begin and lease. + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let lease_req = plugin.a0_lock.as_ref().expect("lock").lease_req; + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![accepted(lease_req)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let depth_req = plugin.a0_lock.as_ref().expect("lock").depth_req; + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![rejected( + depth_req, + "calibrated optical peak u = 1.2 exceeds the lobe ceiling", + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + assert!(plugin.a0_lock.is_none(), "the lock must not keep trying"); + assert!( + plugin.message.contains("lobe ceiling"), + "message: {}", + plugin.message + ); + assert!(plugin.a0_locks.is_empty(), "a rejected lock stores nothing"); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn event_count_point_commands_the_locked_depth_not_a0() { + let folder = temp_folder("ecpoint"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.a0_locks.push(A0LockPoint { + frequency_hz: 1_000.0, + target_a: 0.5, + commanded_a: 0.8333, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: Some(0.0), + high_clip_fraction: Some(0.0), + }); + let mut sink = ControlSink::default(); + + plugin + .set_setting("record_a0_point", json!(true)) + .expect("press"); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let sweep = plugin.sweep.as_ref().expect("event-count sweep"); + assert_eq!(sweep.kind, SweepKind::EventCount); + assert_eq!(sweep.total(), 1); + let lease_req = sweep.lease_req; + + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![accepted(lease_req)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + // The drive is commanded to the locked depth, *not* to a₀ itself. + let commanded = last_commanded_depth(&sink).expect("commanded depth"); + assert!((commanded - 0.833).abs() < 0.002, "commanded {commanded}"); + assert!((plugin.sweep.as_ref().expect("sweep").target_a() - 0.5).abs() < 1e-9); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn event_count_stems_and_sidecars_carry_the_frequency_and_the_lock() { + let folder = temp_folder("ecstem"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.measurement_id = "A1-ecrow".into(); + plugin.frame_width = 4; + plugin.frame_height = 1; + let lock = A0LockPoint { + frequency_hz: 1_000.0, + target_a: 0.5, + commanded_a: 0.8333, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: 1_784_764_800_000, + low_clip_fraction: Some(0.0), + high_clip_fraction: Some(0.0), + }; + plugin.sweep = Some(Sweep { + phase: SweepPhase::Recording, + kind: SweepKind::EventCount, + points: vec![SweepPoint { + commanded_a: 0.8333, + expected_a: 0.5, + }], + lock: Some(lock), + index: 0, + lease_id: LeaseId::new("a1-sweep-test"), + lease_granted: true, + lease_req: 0, + depth_req: 0, + depth_applied: true, + settled_since_ms: None, + settle_deadline_ms: 0, + point_started: true, + last_activity_ms: 0, + stop_requested: false, + }); + + // The stem carries the frequency instead of a sweep-point index. + let mut sink = ControlSink::default(); + plugin.begin_recording(&mut sink, RecRole::EventCount); + let stem = plugin.recording.stem.clone(); + assert!(stem.ends_with("_ec_f1000Hz"), "stem: {stem}"); + + plugin.recording.duration_s = 5; + plugin.recording.start_unix_ms = 1_784_764_800_000; + let path = plugin.write_sidecar().expect("sidecar path"); + let text = std::fs::read_to_string(&path).expect("read sidecar"); + assert!(text.contains("role = \"event-count point\""), "{text}"); + assert!(text.contains("[a0_lock]"), "{text}"); + assert!(text.contains("target_a = 0.5"), "{text}"); + assert!(text.contains("commanded_a = 0.8333"), "{text}"); + assert!(text.contains("converged = true"), "{text}"); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn frequency_tags_are_file_safe() { + assert_eq!(frequency_tag(50.0), "f50Hz"); + assert_eq!(frequency_tag(0.5), "f0p5Hz"); + assert_eq!(frequency_tag(1_200.0), "f1200Hz"); + assert_eq!(frequency_tag(12.345), "f12p345Hz"); + assert_eq!(sanitize_stem(&frequency_tag(0.5)), frequency_tag(0.5)); + } + + #[test] + fn locks_are_one_per_frequency_and_round_trip_through_the_folder() { + let dir = std::env::temp_dir().join(format!("a1-a0-{}", now_unix_ms())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let folder = dir.display().to_string(); + + let mut plugin = StageAA1Plugin { + output_folder: folder.clone(), + ..StageAA1Plugin::default() + }; + let point = |hz: f64, commanded_a: f64| A0LockPoint { + frequency_hz: hz, + target_a: 0.5, + commanded_a, + measured_a: 0.5, + trials: 2, + converged: true, + locked_at_unix_ms: now_unix_ms(), + low_clip_fraction: None, + high_clip_fraction: None, + }; + plugin.store_lock(point(1_000.0, 0.83)).expect("saved"); + plugin.store_lock(point(50.0, 0.52)).expect("saved"); + // Re-locking the same frequency replaces the row rather than appending. + plugin.store_lock(point(1_000.5, 0.86)).expect("saved"); + assert_eq!(plugin.a0_locks.len(), 2); + assert!( + (plugin.a0_locks[0].frequency_hz - 50.0).abs() < 1e-9, + "sorted by frequency" + ); + + let mut other = StageAA1Plugin { + output_folder: folder.clone(), + ..StageAA1Plugin::default() + }; + other.load_a0_locks(); + assert_eq!(other.a0_locks.len(), 2); + let reloaded = other.lock_for_frequency(1_000.0).expect("reloaded lock"); + assert!((reloaded.commanded_a - 0.86).abs() < 1e-9); + assert_eq!(other.a0_locks_dataset().columns.len(), 7); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn settings_discontinuities_keep_the_response_curve() { let mut plugin = StageAA1Plugin::default(); diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 2dc3e51..f60ac8f 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -4802,7 +4802,10 @@ level = 750 reply.outcome ); } - assert!((plugin.depth_a - 1.25).abs() < 1e-9, "sweep drives the depth"); + assert!( + (plugin.depth_a - 1.25).abs() < 1e-9, + "sweep drives the depth" + ); plugin.end_lease(); assert!( diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 1e845eb..95cbc59 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -83,9 +83,20 @@ const SPECTRUM_MIN_SAMPLES: usize = 256; const SPECTRUM_MAX_SAMPLES: usize = 16_384; /// The firmware's default stream rate; the mock mirrors it. const MOCK_RATE_HZ: u32 = 20_000; -/// Trailing samples used for the live optical log-contrast `a`. Sized like the -/// spectrum window so a handful of modulation cycles are always covered. +/// Floor on the trailing samples used for the live optical log-contrast `a`, +/// and the fallback window when no phase-0 markers give a period. Sized like +/// the spectrum window: 16 384 samples ≈ 0.82 s at 20 kSa/s. const CONTRAST_WINDOW_SAMPLES: usize = 16_384; +/// Whole modulation cycles the contrast window is sized to cover. +/// +/// `a` is a *peak-to-peak* quantity, so a window shorter than one cycle sees +/// only an arc of the waveform and under-reports it — and a consumer that +/// divides by the measured `a` (A1's `a₀` lock) then inflates its drive against +/// that bias. A fixed 0.82 s window is below one cycle for every `f < 1.2 Hz`, +/// i.e. exactly the sub-hertz plateau reference the A1 protocol needs. The +/// markers give the period on the same sample clock, so size the window from +/// them instead. +const CONTRAST_WINDOW_CYCLES: f64 = 8.0; const MOCK_BLOCK_SAMPLES: usize = 256; /// Cap on retained phase-0 markers (bounds the overlay + frequency window). const MAX_MARKERS: usize = 4_096; @@ -171,6 +182,16 @@ struct SharedState { /// `Marker` stream frames. Used for the opt-in trigger overlay and to derive /// the modulation frequency. markers: VecDeque, + /// Newest phase-0 marker index seen, retained or already evicted, and the + /// spacing to the one before it. + /// + /// The retained markers alone cannot measure a period longer than the ring: + /// once the ring holds less than one cycle it holds at most one marker, so + /// the mean spacing is undefined exactly where knowing the period matters + /// most. Markers arrive one at a time, so remember the interval as it goes + /// past instead of trying to recover it from what survived eviction. + last_marker_index: Option, + marker_period_estimate: Option, latest: Option, /// Cumulative firmware-side drop counter (latest header value). device_dropped: u32, @@ -218,6 +239,8 @@ impl Default for SharedState { samples: VecDeque::new(), cells: VecDeque::new(), markers: VecDeque::new(), + last_marker_index: None, + marker_period_estimate: None, latest: None, device_dropped: 0, crc_failures: 0, @@ -252,6 +275,10 @@ impl SharedState { self.samples.clear(); self.cells.clear(); self.markers.clear(); + // The sample clock restarts with the segment, so a spacing + // measured across the discontinuity is meaningless. + self.last_marker_index = None; + self.marker_period_estimate = None; self.ring_first_index = first_index; self.rate_hz = rate_hz; } @@ -312,6 +339,12 @@ impl SharedState { { return; // ignore duplicate stamps } + if let Some(previous) = self.last_marker_index { + if sample_index > previous { + self.marker_period_estimate = Some((sample_index - previous) as f64); + } + } + self.last_marker_index = Some(sample_index); self.markers.push_back(sample_index); while self.markers.len() > MAX_MARKERS { self.markers.pop_front(); @@ -319,11 +352,31 @@ impl SharedState { self.last_update_unix_ms = now_unix_ms(); } + /// How many trailing samples the optical log-contrast is estimated over, + /// with the whole modulation cycles that window covers. + /// + /// `a` is peak-to-peak, so the window has to span whole cycles: sized to + /// [`CONTRAST_WINDOW_CYCLES`] of the marker-measured period, floored at + /// [`CONTRAST_WINDOW_SAMPLES`] so nothing gets shorter than today at high + /// `f`, and capped by what the ring actually retains. `covered_cycles` is + /// `None` when there is no period to measure against — then the caller can + /// only fall back to the fixed window and say so. + fn contrast_window(&self) -> (usize, Option) { + let available = self.samples.len(); + let Some(period) = self.marker_period_samples() else { + return (available.min(CONTRAST_WINDOW_SAMPLES), None); + }; + let wanted = (period * CONTRAST_WINDOW_CYCLES).ceil() as usize; + let window = wanted.max(CONTRAST_WINDOW_SAMPLES).min(available); + (window, Some(window as f64 / period)) + } + /// Mean marker spacing in samples, i.e. the modulation period on the device /// clock — the trigger *defining* the frequency. `None` with < 2 markers. fn marker_period_samples(&self) -> Option { if self.markers.len() < 2 { - return None; + // Below one retained cycle only the remembered interval is left. + return self.marker_period_estimate; } let first = *self.markers.front()?; let last = *self.markers.back()?; @@ -1529,8 +1582,8 @@ impl StageAPhotodiodePlugin { /// retarget the sweep and write a wrong `measured_a` into every sidecar. /// /// `None` when there is no valid window or no valid total-power anchor. - fn optical_summary(&self, samples: &VecDeque) -> Option { - self.optical_summary_result(samples).ok() + fn optical_summary(&self, state: &SharedState) -> Option { + self.optical_summary_result(state).ok() } /// [`Self::optical_summary`], keeping the rejection reason so the status @@ -1538,9 +1591,23 @@ impl StageAPhotodiodePlugin { /// showing nothing. fn optical_summary_result( &self, - samples: &VecDeque, + state: &SharedState, ) -> Result { - let start = samples.len().saturating_sub(CONTRAST_WINDOW_SAMPLES); + let samples = &state.samples; + let (window_samples, covered_cycles) = state.contrast_window(); + let rate_hz = f64::from(state.rate_hz.max(1)); + let window_seconds = window_samples as f64 / rate_hz; + // Fail closed below one full cycle: the robust extrema would see an arc + // of the waveform, and `a` would be a phase-dependent under-estimate. A + // consumer that divides by the measured `a` — A1's `a₀` lock — would + // then drive itself up against a bias it cannot see. + if let Some(cycles) = covered_cycles.filter(|cycles| *cycles < 1.0) { + return Err(EstimateError::WindowShorterThanCycle { + covered_cycles: cycles, + window_seconds, + }); + } + let start = samples.len().saturating_sub(window_samples); let window: Vec = samples.iter().skip(start).copied().collect(); let calibration = self.adc_calibration(); // `ContrastGeometry::RejectedComplement` wants the *dark-corrected* @@ -1585,9 +1652,13 @@ impl StageAPhotodiodePlugin { excitation_headroom_volts: estimate.v_min_volts, low_clip_fraction: estimate.low_clip_fraction, high_clip_fraction: estimate.high_clip_fraction, - measured_frequency_hz: None, + // The phase-0 markers are the trigger that *defines* the frequency, + // on the same sample clock as the codes above. + measured_frequency_hz: state.marker_period_samples().map(|period| rate_hz / period), fundamental_phase_rad: None, total_harmonic_distortion: None, + window_seconds: Some(window_seconds), + covered_cycles, }) } @@ -1595,7 +1666,7 @@ impl StageAPhotodiodePlugin { /// keeping the rejection reason so the caller can explain a withheld `a`. fn latest_optical_result(&self) -> Option> { let state = self.shared.lock().ok()?; - (!state.samples.is_empty()).then(|| self.optical_summary_result(&state.samples)) + (!state.samples.is_empty()).then(|| self.optical_summary_result(&state)) } fn control_summary(&self) -> PhotodiodeSummaryV1 { @@ -1606,7 +1677,7 @@ impl StageAPhotodiodePlugin { end_sample_index_exclusive: state.ring_first_index + state.samples.len() as u64, sample_count: state.samples.len() as u64, }); - let optical_summary = self.optical_summary(&state.samples); + let optical_summary = self.optical_summary(&state); let level = self.current_level(&state); let connection = if self.connected() { ConnectionStateV1::Connected { @@ -3347,15 +3418,139 @@ mod tests { /// A clean rejected-port sine: the detector swings around `center` while /// the excitation is its complement against `I_tot`. - fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> VecDeque { + fn rejected_port_samples(center: f64, amplitude: f64, count: usize) -> Vec { (0..count) .map(|i| { let phase = 2.0 * std::f64::consts::PI * (i as f64) * 8.0 / count as f64; - (center + amplitude * phase.sin()).round().clamp(0.0, 4_095.0) as u16 + (center + amplitude * phase.sin()) + .round() + .clamp(0.0, 4_095.0) as u16 }) .collect() } + /// [`rejected_port_samples`] ingested into a ring, with one phase-0 marker + /// per cycle when `mark_cycles` — the estimator sizes its window from them. + fn rejected_port_state( + center: f64, + amplitude: f64, + count: usize, + mark_cycles: bool, + ) -> SharedState { + let mut state = SharedState::default(); + state.ingest( + 0, + 20_000, + 0, + &rejected_port_samples(center, amplitude, count), + ); + if mark_cycles { + // `rejected_port_samples` puts 8 whole cycles in `count` samples. + let period = (count / 8) as u64; + for cycle in 0..8 { + state.push_marker(cycle * period); + } + } + state + } + + /// A slow sine streamed for `total` samples into a ring that only retains + /// `retained` of them, with one phase-0 marker per cycle delivered as the + /// stream goes past — so markers are evicted exactly as they are on the + /// bench when the period outgrows the monitor cache. + fn slow_sine_state(period_samples: u64, retained: usize, total: usize) -> SharedState { + let mut state = SharedState { + cache_seconds: retained as f64 / 20_000.0, + ..Default::default() + }; + let block = 4_000; + let mut index = 0usize; + while index < total { + let end = (index + block).min(total); + let codes: Vec = (index..end) + .map(|i| { + let phase = 2.0 * std::f64::consts::PI * (i as f64) / period_samples as f64; + (1_600.0 + 700.0 * phase.sin()).round() as u16 + }) + .collect(); + state.ingest(index as u64, 20_000, 0, &codes); + let mut marker = index.next_multiple_of(period_samples as usize) as u64; + while (marker as usize) < end { + state.push_marker(marker); + marker += period_samples; + } + index = end; + } + state + } + + #[test] + fn a_window_shorter_than_one_cycle_withholds_a_instead_of_under_reporting_it() { + // `a` is peak-to-peak. Below one full cycle the robust extrema see an + // arc of the sine, so `a` comes out low — and A1's a₀ lock divides by + // it, inflating its drive against a bias it cannot see. Fail closed. + let mut plugin = live_plugin(); + plugin.reference_volts = 3.0; + + // 0.5 Hz at 20 kSa/s = 40 000 samples per cycle; retain 0.6 of one. + let partial = slow_sine_state(40_000, 24_000, 200_000); + let error = plugin + .optical_summary_result(&partial) + .expect_err("a partial cycle must not publish an a"); + assert!( + matches!( + error, + EstimateError::WindowShorterThanCycle { covered_cycles, .. } + if (covered_cycles - 0.6).abs() < 0.05 + ), + "unexpected rejection: {error:?}" + ); + + // Two whole cycles of the same drive: published, and the window is + // reported so a consumer can wait it out before trusting a re-read. + let whole = slow_sine_state(40_000, 80_000, 200_000); + let summary = plugin + .optical_summary(&whole) + .expect("two whole cycles estimate"); + let expected = ((3.0_f64 - (1_600.0 - 700.0) * (3.3 / 4_095.0)) + / (3.0 - (1_600.0 + 700.0) * (3.3 / 4_095.0))) + .ln(); + assert!( + (summary.measured_log_contrast - expected).abs() < 0.02, + "a={} expected~{expected}", + summary.measured_log_contrast + ); + assert!((summary.measured_frequency_hz.expect("markers") - 0.5).abs() < 0.01); + assert!((summary.window_seconds.expect("window") - 4.0).abs() < 0.01); + assert!(summary.covered_cycles.expect("cycles") >= 1.0); + } + + #[test] + fn the_contrast_window_grows_to_cover_whole_cycles_at_low_frequency() { + // A fixed 16 384-sample window is 0.82 s: below one cycle for every + // f < 1.2 Hz, which is where the A1 plateau reference lives. + let fast = rejected_port_state(1_600.0, 700.0, 4_096, true); + let (window, cycles) = fast.contrast_window(); + assert_eq!(window, 4_096, "high f keeps the whole retained ring"); + assert!(cycles.expect("markers") >= 8.0); + + let slow = slow_sine_state(40_000, 400_000, 400_000); + let (window, cycles) = slow.contrast_window(); + assert_eq!( + window, + (CONTRAST_WINDOW_CYCLES as usize) * 40_000, + "the window is sized from the marker period, not fixed" + ); + assert!((cycles.expect("markers") - CONTRAST_WINDOW_CYCLES).abs() < 0.01); + + // Without markers there is no period to size against: fall back to the + // fixed window and report no cycle count rather than guess one. + let mut unmarked = slow_sine_state(40_000, 400_000, 400_000); + unmarked.markers.clear(); + unmarked.marker_period_estimate = None; + assert_eq!(unmarked.contrast_window(), (CONTRAST_WINDOW_SAMPLES, None)); + } + #[test] fn published_contrast_is_the_excitation_contrast_in_both_display_modes() { // The detector sits behind the PBS reject port whatever the operator @@ -3363,14 +3558,12 @@ mod tests { // scientific quantity. A1's amplitude sweep settles on this value. let mut plugin = live_plugin(); plugin.reference_volts = 3.0; - let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + let state = rejected_port_state(1_600.0, 700.0, 4_096, true); plugin.mode = Mode::Raw; - let raw = plugin.optical_summary(&samples).expect("raw display"); + let raw = plugin.optical_summary(&state).expect("raw display"); plugin.mode = Mode::Excitation; - let excitation = plugin - .optical_summary(&samples) - .expect("excitation display"); + let excitation = plugin.optical_summary(&state).expect("excitation display"); assert_eq!(raw.measured_log_contrast, excitation.measured_log_contrast); assert_eq!(raw.calibration.anchor_id, "reference-volts"); @@ -3390,14 +3583,14 @@ mod tests { fn captured_dark_level_reaches_the_estimator_and_is_named() { let mut plugin = live_plugin(); plugin.reference_volts = 3.0; - let samples = rejected_port_samples(1_600.0, 700.0, 4_096); + let state = rejected_port_state(1_600.0, 700.0, 4_096, true); - let undarkened = plugin.optical_summary(&samples).expect("no dark yet"); + let undarkened = plugin.optical_summary(&state).expect("no dark yet"); assert_eq!(undarkened.calibration.dark_id, "dark-none"); assert_eq!(undarkened.calibration.dark_volts, 0.0); plugin.dark_volts = 0.05; - let darkened = plugin.optical_summary(&samples).expect("with dark"); + let darkened = plugin.optical_summary(&state).expect("with dark"); assert_eq!(darkened.calibration.dark_id, "dark-measured"); assert_eq!(darkened.calibration.dark_volts, 0.05); // A DC dark offset is common to the detector samples and to the @@ -3679,7 +3872,7 @@ mod tests { railed.ingest(0, 20_000, 0, &[4_095; 8]); let clipped = plugin.current_level(&railed).expect("still reports"); assert!(clipped.clipped); - assert!(plugin.optical_summary(&railed.samples).is_none()); + assert!(plugin.optical_summary(&railed).is_none()); } #[test] diff --git a/stage-a-io/src/estimator.rs b/stage-a-io/src/estimator.rs index 1c87f44..cfb445f 100644 --- a/stage-a-io/src/estimator.rs +++ b/stage-a-io/src/estimator.rs @@ -105,6 +105,18 @@ pub enum EstimateError { total_power_volts: f64, detector_max_volts: f64, }, + /// The window is shorter than one full modulation cycle, so the robust + /// extrema only see an arc of the waveform and `a` would be a + /// phase-dependent *under*-estimate. + /// + /// Constructed by the caller — [`estimate_contrast`] is given codes, not a + /// period, and sizing the window against the drive is the caller's job. + /// It is in this enum because it belongs with the other fail-closed + /// reasons a consumer has to render. + WindowShorterThanCycle { + covered_cycles: f64, + window_seconds: f64, + }, } impl std::fmt::Display for EstimateError { @@ -131,6 +143,14 @@ impl std::fmt::Display for EstimateError { "total-power anchor {total_power_volts:.4} V is not above the detector \ maximum {detector_max_volts:.4} V; a is undefined" ), + Self::WindowShorterThanCycle { + covered_cycles, + window_seconds, + } => write!( + f, + "the {window_seconds:.2} s window covers only {covered_cycles:.2} modulation \ + cycles; a needs at least one full cycle — raise the cache length" + ), } } } diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index e7e1638..545644b 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -541,6 +541,18 @@ pub struct PhotodiodeOpticalSummaryV1 { pub measured_frequency_hz: Option, pub fundamental_phase_rad: Option, pub total_harmonic_distortion: Option, + /// Duration of the trailing window `measured_log_contrast` was estimated + /// over. A consumer that *commands* a depth and then reads this value back + /// has to wait at least this long, or it averages the previous depth in. + /// Additive in V1: absent from older owners, ignored by older consumers. + #[serde(default)] + pub window_seconds: Option, + /// Whole modulation cycles that window covered, from the phase-0 markers. + /// `a` is peak-to-peak, so below one cycle the owner withholds it entirely + /// rather than publish a phase-dependent under-estimate. `None` when there + /// is no marker period to measure against. + #[serde(default)] + pub covered_cycles: Option, } /// Settled detector level over the newest averaging window, in **raw detector From 3fee3d68493664defae2d90c39cf661a37eead42 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Mon, 27 Jul 2026 21:49:44 +0200 Subject: [PATCH 27/30] =?UTF-8?q?feat(stage-a):=20=E2=9C=A8=20run=20the=20?= =?UTF-8?q?A1=20a=E2=82=80=20frequency=20ladder=20unattended=20on=20one=20?= =?UTF-8?q?lease?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An A1 event-count block is 7-12 frequencies over two or three decades, each one a Find a₀ and a Record a₀ point, repeated over three blocks. ADR 013 left that manual because the ordering decisions are scientific — but they are also expressible, and the checklist wants them frozen in the session plan anyway. Every gap between the two presses was also a gap in which a modulation settings sync could re-apply the operator's own depth on top of the found one. Start frequency sweep runs the whole ladder on ONE modulation lease: per point it retargets the drive frequency, waits for the phase-0 trigger to confirm the new period, runs the unchanged a₀ lock, and records the unchanged event-count point. Both children gained an inherited-lease mode, so they run on the ladder's lease instead of taking their own — which is the substantive guarantee: the operator's drive settings are locked out from the first frequency to the last, so the amplitude provably cannot move between a lock and the point that replays it. `ModulationCommandV1::SetDriveFrequency` is the frequency counterpart of SetOpticalDepth (additive in V1, same scoping: leased only, re-derived through the owner's own drive_command, refused for a manual DAC or constant drive). The owner parks the operator's armed frequency on the first retarget and restores it with the depth when the lease ends, so a finished ladder does not leave the bench on its last point. Robustness, which is most of the work: - the trigger confirms the frequency, not the firmware ACK. A point starts only once enough phase-0 markers at the *new* period agree with the commanded one - retained markers and events are dropped on every frequency change: the measured period is their mean spacing, so keeping them would confirm the new frequency against a mixture of the old drive and the new - pilot windows are dropped with them. Windows frozen at one period are a phase interval of that period; carrying them across would score a point in the wrong window, silently, because a fold always produces something - the plan is validated before the drive moves. The photodiode estimates `a` over one window for the whole ladder, so its lowest frequency decides whether the ladder is measurable at all — checked at the button press, not at the ninth point two hours in - an unreachable a₀, an unconfirmed frequency or a failed recording skips that point and names it in the summary; the remaining decades are still recorded. A refused frequency carries the owner's own wording into the skip The schedule is data: log spacing (|H(f)| is read per decade), ascending / descending / alternating / seeded-random order, and an optional low-frequency reference interleaved every N points. Every point's sidecar gains a [frequency_sweep] section with the executed position, the order and the seed, so a block is interpretable from its files rather than from a notebook. See ADR 014. --- docs/adr/014-stage-a-a1-frequency-ladder.md | 109 ++ docs/features/README.md | 2 +- docs/features/stage-a-a1-event-count.md | 106 +- docs/features/stage-a-a1.md | 3 +- plugins/stage-a-a1/README.md | 4 +- plugins/stage-a-a1/src/runtime.rs | 1511 ++++++++++++++++++- plugins/stage-a-modulation/src/lib.rs | 70 +- stage-a-plugin-contract/src/lib.rs | 13 + 8 files changed, 1769 insertions(+), 49 deletions(-) create mode 100644 docs/adr/014-stage-a-a1-frequency-ladder.md diff --git a/docs/adr/014-stage-a-a1-frequency-ladder.md b/docs/adr/014-stage-a-a1-frequency-ladder.md new file mode 100644 index 0000000..3b1f99e --- /dev/null +++ b/docs/adr/014-stage-a-a1-frequency-ladder.md @@ -0,0 +1,109 @@ +# ADR 014 — Stage-A A1 unattended frequency ladder + +- **Status:** accepted (2026-07-27) +- **Relates to:** ADR 009 (recording coordinator), ADR 010 (amplitude sweep via + leased `SetOpticalDepth`), ADR 012 (the contrast geometry the measured `a` is + defined in), ADR 013 (the per-frequency `a₀` lock), + [Stage-A A1 Exact Event Count](../features/stage-a-a1-event-count.md) + +## Context + +ADR 013 gave the operator two buttons per frequency — *Find a₀* and *Record a₀ +point* — and deliberately left the ladder manual, because "the protocol's +ordering and randomisation decisions are scientific, not mechanical". + +In practice an A1 event-count block is 7–12 frequencies over two or three +decades, each one a lock plus a recording, repeated over three independent +blocks. That is an hour of pressing two buttons in the right order while +watching a status line — and every gap between the two presses is a gap in which +a modulation settings sync can re-apply the operator's own `depth a` on top of +the found one (ADR 013 §3 exists precisely because of this hazard, and only +closes it *within* one point). + +The ordering decisions are scientific, but they are also **expressible**: a +seeded schedule and an interleaved reference cadence are exactly what the A1 +checklist asks to be frozen in the session plan before the block starts. Freezing +them as settings and recording them per point is stronger than leaving them to +be executed by hand and written down afterwards. + +Three things blocked automation: + +1. **A1 could not change the frequency.** The contract exposed `SetOpticalDepth` + but no frequency equivalent, and A1's scoped reach (ADR 007/010) was "the + armed drive's depth while leased". +2. **Nothing could confirm a frequency had arrived.** The firmware ACKs a table + it accepted; the light modulating at that rate is a different claim. +3. **The measured `a` was not trustworthy at the bottom of a ladder.** The + photodiode estimated the peak-to-peak contrast over a fixed 0.82 s window, + under one cycle for every `f < 1.2 Hz` — and the `a₀` lock divides by that + value, so a truncated estimate drives the depth up until it rails. + +## Decision + +**1. `ModulationCommandV1::SetDriveFrequency { frequency_millihz }`** (contract +addition, additive to V1) — the frequency counterpart of `SetOpticalDepth`, with +the same scoping: leased only, re-derived through the same `drive_command()` +builder, rejected when the link is closed or the armed drive has no frequency to +retarget. The owner **parks the operator's armed frequency** on the first +retarget and restores it in `end_lease`, exactly as it already does for the +depth, so a finished ladder does not leave the bench on its last point. + +**2. The ladder is a supervisor, not a third state machine.** `FreqSweep` runs +`AcquiringLease → (per point) SettingFrequency → ConfirmingFrequency → Locking → +Recording → …release`, where *Locking* and *Recording* are the **unchanged** +ADR 013 lock and ADR 010/013 point. Both gained an inherited-lease mode +(`owns_lease: false`): when the ladder starts them they neither acquire nor +release, they run on its lease. + +That is the substantive guarantee: **one lease spans the whole ladder**, so the +operator's drive settings are locked out from the first frequency to the last, +and the "the amplitude cannot change during the recorded interval" property +ADR 013 established for one point now holds across the gap between a lock and +the point that replays it. + +**3. The trigger confirms the frequency.** A point does not start until enough +phase-0 markers *at the new period* agree with the commanded frequency. On every +frequency change the retained markers and events are dropped: the measured +period is their mean spacing, so keeping them would confirm the new frequency +against a mixture of the old drive and the new one. + +**Pilot windows are dropped with them.** Windows frozen at one period are a +phase interval of *that* period; carrying them into another frequency would +score the point in the wrong window — silently, because a fold always produces +something. Re-freezing a pilot per frequency stays the operator's call; the +ladder only guarantees it never reuses a stale one. + +**4. The schedule is data.** Log spacing (a Bode ladder is read per decade), +four orders — ascending, descending, alternating, seeded random — and an +optional low-frequency reference interleaved every N points. The executed +position, the order and the seed go into every point's sidecar +(`[frequency_sweep]`), so a block is interpretable from its files rather than +from a notebook. + +**5. A bad point is skipped, not fatal.** An unreachable `a₀`, an unconfirmed +frequency, or a failed recording skips that frequency and names it in the final +summary; the lock table keeps the failed attempt. The remaining decades are +worth more than a clean abort. Only losing the lease ends the ladder. + +**6. The plan is validated before the drive moves.** The photodiode estimates +`a` over one window for the whole ladder, so its *lowest* frequency decides +whether the ladder is measurable. That, the drivability of `a₀`, the presence of +a trigger, and the destination are all checked at the button press. + +## Consequences + +- A1's scoped hardware reach widens by one parameter: it may retarget the armed + drive's **frequency** as well as its depth, still only while leased, still + through the owner's own builder and validation. Everything else about the + drive remains the operator's. +- The lease is now held for the length of a whole block rather than a point, so + its TTL is sized from the ladder (renewed per point). A lost lease ends the + run — which is the correct failure: without it the drive is no longer + provably A1's. +- Pilot-frozen windows no longer survive a frequency change. A workflow that + relied on freezing one pilot and recording several frequencies against it was + producing wrongly-scored `q_p`; it now falls back to per-fold auto-windows and + says so. +- `a₀` is still not frozen numerically here, the refractory bound is still not + checked, and references are still the operator's. The ladder automates the + mechanical repetition, not the scientific choices. diff --git a/docs/features/README.md b/docs/features/README.md index 8606c60..43305e3 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -11,7 +11,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 photodiode readout from the PDA1 stream port at 20 kSa/s with envelope decimation and a period-synced moving average: raw values or excitation power `I_exc = I_tot − I_pd` as a display choice, plus the excitation log-contrast `a` — always computed in the reject-port complement geometry with a measured dark level, never from the display mode (ADR 012). - [Stage-A A1 Analysis](./stage-a-a1.md) — Stage-A recording coordinator: one-button synchronized camera RAW + photodiode PDQ recording with a config sidecar per `(I_k, f)` measurement, a one-button amplitude sweep (leased `SetOpticalDepth` + settle + per-point recording), plus live rolling-response and response-probability quicklooks. - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. -- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀`, a per-frequency lock table on disk, and a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease. +- [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. diff --git a/docs/features/stage-a-a1-event-count.md b/docs/features/stage-a-a1-event-count.md index f9c4d7e..b10ac64 100644 --- a/docs/features/stage-a-a1-event-count.md +++ b/docs/features/stage-a-a1-event-count.md @@ -1,11 +1,15 @@ # Stage-A A1 Exact Event Count — the `a₀` depth lock - **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) -- **Status:** Built — per-frequency `a₀` lock + one-button event-count point -- **Design:** [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md); builds - on [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (leased - `SetOpticalDepth`) and [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) - (the RAW + PDQ + sidecar coordinator) +- **Status:** Built — per-frequency `a₀` lock, one-button event-count point, and + an unattended frequency ladder that does both at every planned `f` +- **Design:** [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (the + lock) and [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the ladder); + builds on [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (leased + `SetOpticalDepth`), [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md) + (the RAW + PDQ + sidecar coordinator) and + [ADR 012](../adr/012-stage-a-contrast-geometry-is-bench-not-display.md) (the + geometry the measured `a` is defined in) - **Relates to:** [Stage-A A1 Analysis](./stage-a-a1.md), [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md), [Stage-A Photodiode](./stage-a-photodiode.md) @@ -22,7 +26,8 @@ a_0=\ln\!\left(\frac{I_{\mathrm{exc,max}}}{I_{\mathrm{exc,min}}}\right), and hold that **photodiode-measured** value constant while the frequency varies, so event counts per half-cycle are comparable across `f` at equal optical -contrast. `a₀` is a measured log contrast — **never** a DAC-code excursion. +contrast. `a₀` is a measured log contrast — **never** a DAC-code excursion, and +never the reject-port detector's own contrast (ADR 012). ## Why a lock is needed at all @@ -37,6 +42,31 @@ record at the wrong depth. The lock closes that loop: it commands, measures, and corrects until the photodiode reports `a₀`. +## What the measured `a` needs to be worth dividing by + +The lock divides by the measured `a`, so a *biased* measurement is not noise — +it is a systematic push on the drive. Two properties of the photodiode estimate +therefore matter more here than anywhere else, and both are enforced: + +- **Whole cycles.** `a` is peak-to-peak, so its window has to span at least one + full modulation cycle. The photodiode sizes its contrast window from the + phase-0 markers to cover several cycles, and **withholds `a` entirely** below + one. A fixed 0.82 s window — what it used before — is under one cycle for + every `f < 1.2 Hz`, exactly where the A1 plateau reference lives, and would + have under-reported `a` and driven the depth up until it railed. Its length + and cycle count are published as `window_seconds` / `covered_cycles`. +- **A window that has turned over.** A reading taken sooner than one window + after a depth change still contains the old depth. The lock's per-trial dwell + is therefore at least one window (never less than **Sweep settle (s)**), and + its three readings are spaced by half a window so they are not three views of + the same samples. The trial value is their **median**; if they spread by more + than twice the tolerance the operating point is called unsettled and the lock + aborts rather than latching onto a drifting drive. + +If the ladder's lowest frequency needs a longer window than the photodiode's +ring holds, raise its **Cache length**; the refusal says so and names the +seconds needed. + ## The workflow, one frequency at a time Everything up to the references is unchanged and stays the operator's: freeze the @@ -56,9 +86,67 @@ across frequencies is not automated, i.e. off by default). Then: 3. Press **Record a₀ point (event-count)**. A1 re-applies the found depth under a modulation lease, waits for the measured `a` to hold `a₀`, and records one atomic camera RAW + photodiode PDQ + sidecar under one run id. -4. Repeat for the next frequency. Randomising the frequency order, interleaving - the low-frequency reference and repeating independent blocks (three where - practical) are yours — every point is one button press. +4. Repeat for the next frequency. Repeating independent blocks (three where + practical) is yours — every point is one button press. + +Steps 1–4 are what **Start frequency sweep** automates; see below. + +## The frequency ladder (unattended) + +**Start frequency sweep (find a₀ + record per f)** runs the whole ladder on +**one** modulation lease. Per point it retargets the drive's frequency +(`ModulationCommandV1::SetDriveFrequency`), waits for the phase-0 trigger to +actually report the new period, runs the `a₀` lock, and records one atomic +event-count point — then moves on. + +| Control | Meaning | +|---|---| +| Sweep min f / max f (Hz) | ends of the ladder, both included | +| Frequency points | how many, **log-spaced** — `\|H(f)\|` is read per decade | +| Frequency order | ascending / descending / alternating / random (seeded) | +| Random order seed | makes the random schedule reproducible; recorded per point | +| Low-f reference every N points | re-visit the lowest frequency every N points | +| Start frequency sweep | run the ladder | +| Stop | aborts the ladder and whichever child is mid-flight | + +What it guarantees: + +- **One lease for the whole ladder.** The lock and the recording run on the + ladder's lease instead of taking their own, so the operator's drive settings + are locked out from the first frequency to the last — the amplitude provably + cannot move between a lock and the point that replays it. The owner parks the + operator's frequency *and* depth on the first retarget and hands both back + when the lease is released. +- **The trigger confirms the frequency, not the firmware.** An ACK says a table + was accepted; the phase-0 markers say the light is modulating at that rate. + A point only starts once enough markers at the *new* period agree with the + commanded frequency. +- **Nothing from the previous frequency survives.** Retained markers and events + are dropped on every frequency change — the measured period is their mean + spacing, so keeping them would confirm the new frequency against a mixture. + **Pilot windows are dropped too**: windows frozen at one period do not + transfer to another, and scoring a point in the wrong window is a silent + error. Re-freeze a pilot per frequency if you need pilot-frozen windows. +- **A bad point is skipped, not fatal.** A frequency whose `a₀` is unreachable, + whose trigger never confirms, or whose recording fails is skipped and named in + the final summary; the remaining decades are still recorded. The lock table + keeps the failed attempt. +- **The plan is checked before the drive moves.** The lowest planned frequency + decides whether the photodiode can measure `a` at all, so it is checked up + front — not at the ninth point, two hours in. + +Every point's sidecar gains a `[frequency_sweep]` section: `min_f`, `max_f`, +`planned_points`, the position in the **executed** order, the order name, the +seed, whether the point is an interleaved reference, and the requested +frequency (`[trigger]` carries what the markers measured). + +### What the ladder still does not do + +The flux point `I_k`, the camera configuration, ROI/mask, pedestal, bias set, +gates, the `I_tot` anchor, the zero-depth background, the pilot, and repeating +independent blocks stay the operator's. `a₀` itself is an operator input, and +the refractory condition `2·f·a₀/C ≪ 1/τ_refr` is **not** checked — verify it at +your highest planned frequency when you pick `a₀`. ## Controls diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index dec9534..b230c53 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -5,6 +5,7 @@ - **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button press forwarding), + [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the unattended ladder), [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count `a₀` lock) - **Automation roadmap:** [Stage-A A1 Automation](./stage-a-a1-automation.md) @@ -47,7 +48,7 @@ folder. A1 makes each recording one button press: | Record pilot | records a bright reference (`…_pilot`) **and** freezes the ON/OFF windows for the row from the live signal | | Record background | records an unmodulated reference (`…_background`) **and** captures the false-response floor `q0` | | Stop (abort recording / sweep) | finalize the current recording early; during a sweep also aborts the remaining points | -| a₀ / Find a₀ / Record a₀ point | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep — see [its brief](./stage-a-a1-event-count.md) | +| a₀ / Find a₀ / Record a₀ point / Start frequency sweep | the **exact-event-count** workflow: hold one *measured* depth `a₀` across the frequency sweep, by hand or as an unattended ladder — see [its brief](./stage-a-a1-event-count.md) | The record and sweep buttons stay **disabled until an output folder is selected**. diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 4ab0638..11e3e4d 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -72,6 +72,8 @@ pixels come from the augur-rs camera config. See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the full brief, [ADR 009](../../docs/adr/009-stage-a-a1-recording-coordinator.md) for the coordinator design, [docs/features/stage-a-a1-event-count.md](../../docs/features/stage-a-a1-event-count.md) plus -[ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, and +[ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, +[ADR 014](../../docs/adr/014-stage-a-a1-frequency-ladder.md) for the unattended +frequency ladder, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index e32636e..209bdb9 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -127,6 +127,19 @@ const FREQUENCY_MATCH_FRACTION: f64 = 0.01; /// Lock table persisted in the output folder, so found depths survive a restart. const A0_LOCK_FILE: &str = "a0_locks.json"; +/// Frequency points a single run may visit, before the interleaved references. +const FREQ_SWEEP_MAX_POINTS: usize = 64; +/// How long the frequency sweep waits for the phase-0 trigger to report the +/// frequency it just commanded, before it gives that point up. +/// +/// The drive is a firmware table rebuild plus however long the camera takes to +/// deliver two markers at the new period — at 0.1 Hz that is 20 s on its own. +const FREQ_CONFIRM_BASE_MS: u64 = 20_000; +/// Marker periods that must elapse at the *new* frequency before the sweep +/// believes the measured period. Below this the mean spacing is still a mixture +/// of the old and the new drive. +const FREQ_CONFIRM_CYCLES: f64 = 4.0; + /// Absolute/relative tolerance for "the measured `a` reached the sweep target". fn sweep_tolerance(target_a: f64) -> f64 { (target_a * 0.10).max(0.05) @@ -379,6 +392,11 @@ struct Sweep { lease_id: LeaseId, lease_granted: bool, lease_req: u64, + /// False when the lease belongs to an enclosing run (the frequency sweep): + /// then this run neither acquires nor releases it, so the operator's drive + /// settings stay locked out across the whole ladder rather than only + /// between its points. + owns_lease: bool, depth_req: u64, depth_applied: bool, /// Instant the measured `a` first satisfied the tolerance, for the dwell. @@ -458,6 +476,8 @@ struct A0Lock { lease_id: LeaseId, lease_granted: bool, lease_req: u64, + /// See [`Sweep::owns_lease`]. + owns_lease: bool, depth_req: u64, depth_applied: bool, last_activity_ms: u64, @@ -484,6 +504,121 @@ struct A0LockPoint { high_clip_fraction: Option, } +/// Order the planned frequencies are actually visited in. +/// +/// A Bode ladder recorded strictly low-to-high confounds frequency with +/// everything that drifts monotonically during the block — bleaching, thermal +/// drift of the Pockels bias, source ageing. The A1 checklist therefore asks +/// for a randomised or alternating schedule, and for the seed to be part of the +/// frozen session plan; both are reproduced in the sidecar. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +enum FreqOrder { + #[default] + Ascending, + Descending, + /// Lowest, highest, second lowest, second highest, … — a deterministic + /// alternation that decorrelates frequency from time without a seed. + Alternating, + /// Seeded shuffle; the seed is an operator setting and is recorded. + Random, +} + +impl FreqOrder { + fn from_index(index: u64) -> Self { + match index { + 1 => Self::Descending, + 2 => Self::Alternating, + 3 => Self::Random, + _ => Self::Ascending, + } + } + + fn index(self) -> u64 { + match self { + Self::Ascending => 0, + Self::Descending => 1, + Self::Alternating => 2, + Self::Random => 3, + } + } + + fn label(self) -> &'static str { + match self { + Self::Ascending => "ascending", + Self::Descending => "descending", + Self::Alternating => "alternating", + Self::Random => "random", + } + } +} + +/// One stop of the frequency sweep. +#[derive(Debug, Clone, Copy, PartialEq)] +struct FreqSweepPoint { + frequency_hz: f64, + /// True for the interleaved low-frequency reference repeats, which exist to + /// expose drift across the block rather than to add a new frequency. + is_reference: bool, +} + +/// Where the multi-frequency run is within its per-point cycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FreqSweepPhase { + /// AcquireLease sent to the modulation owner; waiting for the grant. + AcquiringLease, + /// SetDriveFrequency for the current point sent; waiting for Applied. + SettingFrequency, + /// Waiting for the phase-0 trigger to actually report the new period. + ConfirmingFrequency, + /// The `a₀` lock owns this phase. + Locking, + /// The one-point event-count sweep owns this phase. + Recording, +} + +/// One "find a₀ and record a point at every frequency" run. +/// +/// It is a supervisor, not a third copy of the machinery: per point it +/// retargets the leased drive's frequency, waits for the trigger to confirm it, +/// then hands off to the unchanged `a₀` lock and the unchanged event-count +/// point — both running on *this* run's lease, so the operator's drive settings +/// stay locked out from the first frequency to the last. +struct FreqSweep { + phase: FreqSweepPhase, + points: Vec, + index: usize, + lease_id: LeaseId, + lease_granted: bool, + lease_req: u64, + freq_req: u64, + freq_applied: bool, + /// Give-up deadline for the trigger to confirm the commanded frequency. + confirm_deadline_ms: u64, + /// Why the current point is being given up, when that was decided in a + /// service reply rather than in the tick. Carries the owner's own wording + /// through to the skip message instead of replacing it with a timeout. + skip_reason: Option, + /// Points whose `a₀` could not be locked or whose recording failed. Kept + /// and reported rather than aborting the ladder: the remaining frequencies + /// are still worth having, and the lock table already carries the detail. + failed: Vec, + recorded: usize, + order: FreqOrder, + seed: u64, + last_activity_ms: u64, + stop_requested: bool, +} + +impl FreqSweep { + fn point(&self) -> Option { + self.points.get(self.index).copied() + } + + fn frequency_hz(&self) -> f64 { + self.point().map(|point| point.frequency_hz).unwrap_or(0.0) + } +} + /// On-disk form of the per-frequency lock table. #[derive(Debug, Clone, Default, Serialize, serde::Deserialize)] struct A0LockTable { @@ -564,6 +699,20 @@ pub struct StageAA1Plugin { /// Latched by the Record a₀ point button, consumed next control tick. a0_point_pending: bool, a0_lock: Option, + // -- multi-frequency run over the a₀ ladder -- + /// Frequency range and resolution of the planned ladder. Log-spaced: a Bode + /// ladder is read per decade, not per hertz. + min_f: f64, + max_f: f64, + freq_count: u32, + freq_order: FreqOrder, + freq_seed: u64, + /// Insert the lowest planned frequency again after every N points, so drift + /// across the block shows up as a disagreement between its repeats. 0 = off. + freq_reference_every: u32, + /// Latched by the Start frequency sweep button, consumed next control tick. + freq_sweep_pending: bool, + freq_sweep: Option, /// One converged (or attempted) lock per frequency, newest per frequency /// wins; mirrored to `a0_locks.json` in the output folder. a0_locks: Vec, @@ -580,6 +729,7 @@ pub struct StageAA1Plugin { press_record_point: PressLatch, press_clear_curve: PressLatch, press_find_a0: PressLatch, + press_freq_sweep: PressLatch, press_record_a0: PressLatch, press_clear_a0: PressLatch, } @@ -629,6 +779,14 @@ impl Default for StageAA1Plugin { a0_lock_pending: false, a0_point_pending: false, a0_lock: None, + min_f: 1.0, + max_f: 100.0, + freq_count: 7, + freq_order: FreqOrder::Alternating, + freq_seed: 1, + freq_reference_every: 0, + freq_sweep_pending: false, + freq_sweep: None, a0_locks: Vec::new(), loaded_locks_folder: None, press_start: PressLatch::default(), @@ -640,6 +798,7 @@ impl Default for StageAA1Plugin { press_record_point: PressLatch::default(), press_clear_curve: PressLatch::default(), press_find_a0: PressLatch::default(), + press_freq_sweep: PressLatch::default(), press_record_a0: PressLatch::default(), press_clear_a0: PressLatch::default(), } @@ -1601,7 +1760,7 @@ impl StageAA1Plugin { "Sweep: acquiring modulation lease for {} points…", points.len() ); - self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, message); + self.begin_leased_sweep(context, SweepKind::Amplitude, points, None, None, message); } /// Shared entry point for both leased recording runs (amplitude sweep and @@ -1613,6 +1772,7 @@ impl StageAA1Plugin { kind: SweepKind, points: Vec, lock: Option, + inherited_lease: Option, message: String, ) { if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { @@ -1636,12 +1796,18 @@ impl StageAA1Plugin { return; } let now_ms = now_unix_ms(); - let lease_id = LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))); - let ttl_ms = self.sweep_lease_ttl_ms(points.len()); - let request = - self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); - let lease_req = request.request_id; - context.request_service(&request); + let owns_lease = inherited_lease.is_none(); + let lease_id = inherited_lease.unwrap_or_else(|| { + LeaseId::new(format!("a1-sweep-{}", format_compact_utc(now_ms / 1_000))) + }); + let mut lease_req = 0; + if owns_lease { + let ttl_ms = self.sweep_lease_ttl_ms(points.len()); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + lease_req = request.request_id; + context.request_service(&request); + } self.sweep = Some(Sweep { phase: SweepPhase::AcquiringLease, kind, @@ -1649,8 +1815,11 @@ impl StageAA1Plugin { lock, index: 0, lease_id, - lease_granted: false, + // An inherited lease is already granted; the first tick goes + // straight to retargeting the depth. + lease_granted: !owns_lease, lease_req, + owns_lease, depth_req: 0, depth_applied: false, settled_since_ms: None, @@ -1668,7 +1837,11 @@ impl StageAA1Plugin { /// which also locks the operator's drive settings out for the whole point, so /// the amplitude provably cannot change during the recorded interval — and the /// point is then recorded through the same coordinator as every other run. - fn begin_a0_point(&mut self, context: &mut impl RecordingControl) { + fn begin_a0_point( + &mut self, + context: &mut impl RecordingControl, + inherited_lease: Option, + ) { let Some(hz) = self.frequency_hz() else { self.message = "No modulation frequency yet — arm the drive first".into(); return; @@ -1690,13 +1863,20 @@ impl StageAA1Plugin { lock.commanded_a, lock.target_a ); - self.begin_leased_sweep(context, SweepKind::EventCount, points, Some(lock), message); + self.begin_leased_sweep( + context, + SweepKind::EventCount, + points, + Some(lock), + inherited_lease, + message, + ); } /// Release the modulation lease (if held) and clear the sweep. fn finish_sweep(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(sweep) = self.sweep.take() { - if sweep.lease_granted { + if sweep.owns_lease && sweep.lease_granted { let request = self.modulation_request( ModulationCommandV1::ReleaseLease { safe_off: false, @@ -1763,7 +1943,7 @@ impl StageAA1Plugin { if std::mem::take(&mut self.sweep_pending) { self.begin_sweep(context); } else if std::mem::take(&mut self.a0_point_pending) { - self.begin_a0_point(context); + self.begin_a0_point(context, None); } return; } @@ -2061,7 +2241,11 @@ impl StageAA1Plugin { } /// Kick off the closed-loop `a₀` lock at the current frequency. - fn begin_a0_lock(&mut self, context: &mut impl RecordingControl) { + fn begin_a0_lock( + &mut self, + context: &mut impl RecordingControl, + inherited_lease: Option, + ) { if self.recording.is_active() || self.sweep.is_some() || self.a0_lock.is_some() { self.message = "A recording, sweep or a₀ lock is already running".into(); return; @@ -2106,12 +2290,18 @@ impl StageAA1Plugin { .map(|lock| lock.commanded_a) .unwrap_or(target); let now_ms = now_unix_ms(); - let lease_id = LeaseId::new(format!("a1-a0-{}", format_compact_utc(now_ms / 1_000))); - let ttl_ms = self.a0_lock_lease_ttl_ms(); - let request = - self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); - let lease_req = request.request_id; - context.request_service(&request); + let owns_lease = inherited_lease.is_none(); + let lease_id = inherited_lease.unwrap_or_else(|| { + LeaseId::new(format!("a1-a0-{}", format_compact_utc(now_ms / 1_000))) + }); + let mut lease_req = 0; + if owns_lease { + let ttl_ms = self.a0_lock_lease_ttl_ms(); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + lease_req = request.request_id; + context.request_service(&request); + } self.a0_lock = Some(A0Lock { phase: A0LockPhase::AcquiringLease, target_a: target, @@ -2125,17 +2315,25 @@ impl StageAA1Plugin { window_ms: 0, deadline_ms: 0, lease_id, - lease_granted: false, + lease_granted: !owns_lease, lease_req, + owns_lease, depth_req: 0, depth_applied: false, last_activity_ms: now_ms, stop_requested: false, }); - self.message = format!( - "a₀ lock at {}: acquiring the modulation lease…", - frequency_label(hz) - ); + self.message = if owns_lease { + format!( + "a₀ lock at {}: acquiring the modulation lease…", + frequency_label(hz) + ) + } else { + format!( + "a₀ lock at {}: trimming the drive depth…", + frequency_label(hz) + ) + }; } /// Renew the lease and command the current trial's depth. @@ -2181,7 +2379,7 @@ impl StageAA1Plugin { /// the event-count point that follows records at `a₀`. fn finish_a0_lock(&mut self, context: &mut impl RecordingControl, message: String) { if let Some(lock) = self.a0_lock.take() { - if lock.lease_granted { + if lock.owns_lease && lock.lease_granted { let request = self.modulation_request( ModulationCommandV1::ReleaseLease { safe_off: false, @@ -2428,7 +2626,7 @@ impl StageAA1Plugin { fn drive_a0_lock(&mut self, context: &mut impl RecordingControl) { if self.a0_lock.is_none() { if std::mem::take(&mut self.a0_lock_pending) { - self.begin_a0_lock(context); + self.begin_a0_lock(context, None); } return; } @@ -2554,13 +2752,584 @@ impl StageAA1Plugin { lock.last_activity_ms = now_unix_ms(); } } - // The owner refuses a depth its calibrated drive cannot express - // (lobe ceiling, DAC limit) — that *is* the "a₀ unreachable at - // this operating point" answer, so surface its wording verbatim. - PluginServiceOutcome::Rejected { message, .. } => abort( - self, - format!("a₀ lock aborted: the drive rejected the commanded depth: {message}"), - ), + // The owner refuses a depth its calibrated drive cannot express + // (lobe ceiling, DAC limit) — that *is* the "a₀ unreachable at + // this operating point" answer, so surface its wording verbatim. + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("a₀ lock aborted: the drive rejected the commanded depth: {message}"), + ), + } + true + } else { + false + } + } + + // ---- multi-frequency a₀ ladder ----------------------------------------- + + /// The planned frequency ladder, log-spaced and inclusive of both ends. + /// + /// Log spacing because `|H(f)|` is read per decade: a linear ladder spends + /// most of its points where the response is flat and none where it rolls + /// off. + fn planned_frequencies(&self) -> Vec { + let count = self.freq_count.clamp(1, FREQ_SWEEP_MAX_POINTS as u32) as usize; + if count == 1 { + return vec![self.min_f]; + } + let (low, high) = (self.min_f.ln(), self.max_f.ln()); + (0..count) + .map(|index| (low + (high - low) * index as f64 / (count - 1) as f64).exp()) + .collect() + } + + /// The planned ladder in the order it will actually be visited, with the + /// interleaved low-frequency reference repeats inserted. + fn freq_sweep_points(&self) -> Vec { + let mut ladder = self.planned_frequencies(); + match self.freq_order { + FreqOrder::Ascending => {} + FreqOrder::Descending => ladder.reverse(), + FreqOrder::Alternating => { + // Lowest, highest, second lowest, second highest, … + let mut out = Vec::with_capacity(ladder.len()); + let (mut low, mut high) = (0usize, ladder.len()); + while low < high { + out.push(ladder[low]); + low += 1; + if low < high { + high -= 1; + out.push(ladder[high]); + } + } + ladder = out; + } + FreqOrder::Random => { + // A seeded Fisher-Yates with a small xorshift, so the executed + // order is reproducible from the seed recorded in the sidecar. + let mut state = self.freq_seed.max(1); + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for index in (1..ladder.len()).rev() { + ladder.swap(index, (next() % (index as u64 + 1)) as usize); + } + } + } + let reference_hz = self.planned_frequencies().first().copied(); + let every = self.freq_reference_every as usize; + let mut points = Vec::with_capacity(ladder.len() * 2); + for (visited, frequency_hz) in ladder.into_iter().enumerate() { + points.push(FreqSweepPoint { + frequency_hz, + is_reference: false, + }); + // Interleave the low-frequency reference so drift across the block + // shows up as a disagreement between its repeats (A1 checklist, + // "interleave a low-frequency reference to expose drift"). + if let Some(reference_hz) = reference_hz.filter(|_| every > 0) { + if (visited + 1) % every == 0 { + points.push(FreqSweepPoint { + frequency_hz: reference_hz, + is_reference: true, + }); + } + } + } + points + } + + /// Lease TTL for the whole ladder: every point pays a lock and a recording. + fn freq_sweep_lease_ttl_ms(&self, remaining_points: usize) -> u64 { + let per_point_ms = self + .a0_lock_lease_ttl_ms() + .saturating_add(self.sweep_lease_ttl_ms(1)) + .saturating_add(FREQ_CONFIRM_BASE_MS); + (remaining_points as u64) + .saturating_mul(per_point_ms) + .saturating_add(60_000) + } + + /// Kick off the multi-frequency run: validate the whole plan, then lease. + /// + /// Everything checkable is checked *here*, before the drive moves: a plan + /// that cannot work at its lowest frequency should say so in a message, not + /// two hours into a block. + fn begin_freq_sweep(&mut self, context: &mut impl RecordingControl) { + if self.recording.is_active() + || self.sweep.is_some() + || self.a0_lock.is_some() + || self.freq_sweep.is_some() + { + self.message = "A recording, sweep or a₀ lock is already running".into(); + return; + } + if self.output_folder.trim().is_empty() { + self.message = "Set an output folder before sweeping the frequency".into(); + return; + } + if self.measurement_id.trim().is_empty() { + self.message = "Set a measurement id before sweeping the frequency".into(); + return; + } + if !self.modulation_connected() { + self.message = "Modulation owner is not connected — cannot drive the frequency".into(); + return; + } + // Written through `partial_cmp` so a NaN from the settings drag is + // rejected rather than silently passing a negated comparison. + let range_ok = self.min_f.partial_cmp(&0.0) == Some(std::cmp::Ordering::Greater) + && matches!( + self.max_f.partial_cmp(&self.min_f), + Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + ); + if !range_ok { + self.message = "Frequency sweep needs 0 < min f ≤ max f".into(); + return; + } + if self.measured_a().is_none() { + self.message = + "No photodiode-measured a — connect the photodiode and anchor I_tot first".into(); + return; + } + let target = self.a0_target; + if !(COMMANDED_A_MIN..=COMMANDED_A_MAX).contains(&target) { + self.message = format!( + "a₀ = {target:.3} is outside the drivable {COMMANDED_A_MIN}..={COMMANDED_A_MAX}" + ); + return; + } + // The photodiode estimates `a` over one window for all frequencies, so + // the *lowest* planned frequency decides whether the ladder is + // measurable at all. Refuse the plan, not its 9th point. + if let Err(reason) = self.optical_window_covers_a_cycle(self.min_f) { + self.message = format!("Frequency sweep refused at its lowest point: {reason}"); + return; + } + if !self.is_marker_anchored() { + // Without the phase-0 trigger there is nothing that can confirm the + // drive actually reached a commanded frequency, and the fold has no + // anchor either. + self.message = "No phase-0 trigger markers — the sweep cannot confirm a commanded \ + frequency. Enable Live analysis and check EXT_TRIGGER" + .into(); + return; + } + let points = self.freq_sweep_points(); + if points.is_empty() { + self.message = "Nothing to sweep: the frequency ladder has no points".into(); + return; + } + let now_ms = now_unix_ms(); + let lease_id = LeaseId::new(format!("a1-fsweep-{}", format_compact_utc(now_ms / 1_000))); + let ttl_ms = self.freq_sweep_lease_ttl_ms(points.len()); + let request = + self.modulation_request(ModulationCommandV1::AcquireLease { ttl_ms }, &lease_id); + let lease_req = request.request_id; + context.request_service(&request); + let total = points.len(); + self.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::AcquiringLease, + points, + index: 0, + lease_id, + lease_granted: false, + lease_req, + freq_req: 0, + freq_applied: false, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: self.freq_order, + seed: self.freq_seed, + last_activity_ms: now_ms, + stop_requested: false, + }); + self.message = format!( + "Frequency sweep: acquiring the modulation lease for {total} points ({} order)…", + self.freq_order.label() + ); + } + + /// Release the ladder's lease (if this run holds it) and clear the sweep. + /// + /// `safe_off = false` as everywhere else: stopping the drive is the owner's + /// lease-expiry job, not a sweep's. Releasing does hand the operator's own + /// frequency and depth back, because the owner parks them on the first + /// retarget. + fn finish_freq_sweep(&mut self, context: &mut impl RecordingControl, message: String) { + if let Some(sweep) = self.freq_sweep.take() { + if sweep.lease_granted { + let request = self.modulation_request( + ModulationCommandV1::ReleaseLease { + safe_off: false, + reason: "a1 frequency sweep finished".into(), + }, + &sweep.lease_id, + ); + context.request_service(&request); + } + } + self.message = message; + } + + /// Renew the ladder's lease and retarget the drive at the current point. + fn send_freq_sweep_frequency(&mut self, context: &mut impl RecordingControl) { + let Some(sweep) = self.freq_sweep.as_ref() else { + return; + }; + let lease_id = sweep.lease_id.clone(); + let remaining = sweep.points.len().saturating_sub(sweep.index); + let hz = sweep.frequency_hz(); + let (index, total) = (sweep.index, sweep.points.len()); + let is_reference = sweep.point().is_some_and(|point| point.is_reference); + + let ttl_ms = self.freq_sweep_lease_ttl_ms(remaining); + let renew = self.modulation_request(ModulationCommandV1::RenewLease { ttl_ms }, &lease_id); + context.request_service(&renew); + let request = self.modulation_request( + ModulationCommandV1::SetDriveFrequency { + frequency_millihz: (hz * 1_000.0).round().max(0.0) as u64, + }, + &lease_id, + ); + let freq_req = request.request_id; + context.request_service(&request); + + // The retained markers and events belong to the *previous* frequency: + // the measured period is their mean spacing, so leaving them in place + // would confirm the new frequency against a mixture of the two. The + // pilot windows are frozen at a phase of the old period and are not + // transferable either — a point recorded against them would be scored + // in the wrong window. + self.camera_markers_us.clear(); + self.camera_events.clear(); + self.fold_cache.replace(None); + self.pilot_windows = None; + + let now_ms = now_unix_ms(); + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::SettingFrequency; + sweep.freq_req = freq_req; + sweep.freq_applied = false; + sweep.skip_reason = None; + sweep.last_activity_ms = now_ms; + } + self.message = format!( + "Frequency sweep {}/{total}: retargeting the drive to {}{}…", + index + 1, + frequency_label(hz), + if is_reference { " (reference)" } else { "" }, + ); + } + + /// Give up on the current point and move to the next one. + /// + /// A frequency that cannot be locked or recorded does not end the ladder: + /// the remaining points are still worth having, and the failure is already + /// in the lock table. It is reported in the final summary. + fn fail_freq_sweep_point(&mut self, context: &mut impl RecordingControl, reason: String) { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.failed.push(hz); + } + self.message = format!( + "Frequency sweep: skipping {} — {reason}", + frequency_label(hz) + ); + self.advance_freq_sweep(context); + } + + /// Move to the next ladder point, or finish with a summary. + fn advance_freq_sweep(&mut self, context: &mut impl RecordingControl) { + let done = match self.freq_sweep.as_mut() { + Some(sweep) => { + sweep.index += 1; + sweep.index >= sweep.points.len() + } + None => return, + }; + if !done { + self.send_freq_sweep_frequency(context); + return; + } + let (recorded, failed, total, order, seed) = self + .freq_sweep + .as_ref() + .map(|sweep| { + ( + sweep.recorded, + sweep.failed.clone(), + sweep.points.len(), + sweep.order, + sweep.seed, + ) + }) + .unwrap_or_default(); + let mut message = format!( + "Frequency sweep complete: {recorded}/{total} points recorded ({} order, seed {seed})", + order.label() + ); + if !failed.is_empty() { + let list = failed + .iter() + .map(|hz| frequency_label(*hz)) + .collect::>() + .join(", "); + message.push_str(&format!( + " — {} skipped: {list}. See the a₀ lock table", + failed.len() + )); + } + self.finish_freq_sweep(context, message); + } + + /// Advance the multi-frequency run one control tick. Runs before the lock + /// and the point sweep, so a child it starts runs on the same tick. + fn drive_freq_sweep(&mut self, context: &mut impl RecordingControl) { + if self.freq_sweep.is_none() { + if std::mem::take(&mut self.freq_sweep_pending) { + self.begin_freq_sweep(context); + } + return; + } + self.freq_sweep_pending = false; + let now_ms = now_unix_ms(); + let (phase, stop_requested, lease_granted, freq_applied, last_activity_ms, index, total) = { + let sweep = self.freq_sweep.as_ref().expect("sweep checked above"); + ( + sweep.phase, + sweep.stop_requested, + sweep.lease_granted, + sweep.freq_applied, + sweep.last_activity_ms, + sweep.index, + sweep.points.len(), + ) + }; + // A stop propagates into whichever child is running; the ladder ends + // once that child has let go. + if stop_requested { + if let Some(lock) = self.a0_lock.as_mut() { + lock.stop_requested = true; + return; + } + if let Some(sweep) = self.sweep.as_mut() { + sweep.stop_requested = true; + return; + } + let message = if self.message.is_empty() { + "Frequency sweep stopped".into() + } else { + self.message.clone() + }; + self.finish_freq_sweep(context, message); + return; + } + match phase { + FreqSweepPhase::AcquiringLease => { + if lease_granted { + self.send_freq_sweep_frequency(context); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_freq_sweep( + context, + "Frequency sweep aborted: timed out acquiring the modulation lease".into(), + ); + } + } + FreqSweepPhase::SettingFrequency => { + // A refused frequency is a property of this point, not of the + // ladder; the owner already said why. + if let Some(reason) = self + .freq_sweep + .as_mut() + .and_then(|sweep| sweep.skip_reason.take()) + { + self.fail_freq_sweep_point(context, reason); + } else if freq_applied { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // Confirming needs whole cycles at the *new* period, so the + // budget has to scale with it: 4 cycles at 0.1 Hz is 40 s. + let cycles_ms = if hz > 0.0 { + (FREQ_CONFIRM_CYCLES / hz * 1_000.0).ceil() as u64 + } else { + 0 + }; + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::ConfirmingFrequency; + sweep.confirm_deadline_ms = + now_ms.saturating_add(FREQ_CONFIRM_BASE_MS.max(cycles_ms * 3)); + } + self.message = format!( + "Frequency sweep {}/{total}: waiting for the trigger to report {}…", + index + 1, + frequency_label(hz), + ); + } else if now_ms.saturating_sub(last_activity_ms) > REPLY_TIMEOUT_MS { + self.finish_freq_sweep( + context, + "Frequency sweep aborted: timed out retargeting the drive frequency".into(), + ); + } + } + FreqSweepPhase::ConfirmingFrequency => { + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // The trigger *defines* the frequency, so the point only starts + // once the markers say the drive is really there — an ACK from + // the firmware says the table was accepted, not that the light + // is modulating at that rate. Enough markers must have arrived + // at the new period for their mean spacing to mean anything. + let enough_markers = self.camera_markers_us.len() as f64 >= FREQ_CONFIRM_CYCLES; + let confirmed = enough_markers + && self + .frequency_hz() + .is_some_and(|measured| same_frequency(measured, hz)); + let deadline = self + .freq_sweep + .as_ref() + .map(|sweep| sweep.confirm_deadline_ms) + .unwrap_or_default(); + if confirmed { + if let Err(reason) = self.optical_window_covers_a_cycle(hz) { + self.fail_freq_sweep_point(context, reason); + return; + } + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::Locking; + } + let lease = self.freq_sweep.as_ref().map(|sweep| sweep.lease_id.clone()); + self.begin_a0_lock(context, lease); + if self.a0_lock.is_none() { + // `begin_a0_lock` refused and said why; keep its wording. + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + } + } else if now_ms >= deadline { + let measured = self + .frequency_hz() + .map_or_else(|| "—".into(), frequency_label); + self.fail_freq_sweep_point( + context, + format!( + "the trigger never reported it (measured {measured} from {} markers)", + self.camera_markers_us.len() + ), + ); + } + } + FreqSweepPhase::Locking => { + if self.a0_lock.is_some() { + return; + } + let hz = self + .freq_sweep + .as_ref() + .map(FreqSweep::frequency_hz) + .unwrap_or_default(); + // A non-converged lock is stored but never arms a recording, so + // `armed_lock` is the single question worth asking here. + if self.armed_lock().is_none() { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + return; + } + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.phase = FreqSweepPhase::Recording; + } + let lease = self.freq_sweep.as_ref().map(|sweep| sweep.lease_id.clone()); + self.begin_a0_point(context, lease); + if self.sweep.is_none() { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + return; + } + self.message = format!( + "Frequency sweep {}/{total}: recording the a₀ point at {}…", + index + 1, + frequency_label(hz), + ); + } + FreqSweepPhase::Recording => { + if self.sweep.is_some() || self.recording.is_active() { + return; + } + if self.recording_completed_ok { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.recorded += 1; + } + self.advance_freq_sweep(context); + } else { + let reason = self.message.clone(); + self.fail_freq_sweep_point(context, reason); + } + } + } + } + + /// Routes modulation-service replies belonging to the frequency sweep. + fn on_freq_sweep_reply(&mut self, reply: &PluginServiceReply) -> bool { + let Some((lease_req, freq_req)) = self + .freq_sweep + .as_ref() + .map(|sweep| (sweep.lease_req, sweep.freq_req)) + else { + return false; + }; + let abort = |this: &mut Self, message: String| { + this.message = message; + if let Some(sweep) = this.freq_sweep.as_mut() { + sweep.stop_requested = true; + } + }; + if reply.request_id == lease_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.lease_granted = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + PluginServiceOutcome::Rejected { message, .. } => abort( + self, + format!("Frequency sweep aborted: modulation lease rejected: {message}"), + ), + } + true + } else if reply.request_id == freq_req { + match &reply.outcome { + PluginServiceOutcome::Accepted { .. } => { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.freq_applied = true; + sweep.last_activity_ms = now_unix_ms(); + } + } + // A refused frequency is a property of this point, not of the + // ladder: skip it and keep the remaining decades. The skip runs + // on the next tick, through the one path that advances the + // ladder, carrying the owner's wording. + PluginServiceOutcome::Rejected { message, .. } => { + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.skip_reason = + Some(format!("the drive rejected the frequency: {message}")); + } + } } true } else { @@ -2648,7 +3417,10 @@ impl StageAA1Plugin { } fn on_service_reply(&mut self, reply: &PluginServiceReply) { - if self.on_sweep_reply(reply) || self.on_a0_lock_reply(reply) { + if self.on_sweep_reply(reply) + || self.on_a0_lock_reply(reply) + || self.on_freq_sweep_reply(reply) + { return; } let response = match &reply.outcome { @@ -2870,6 +3642,19 @@ impl StageAA1Plugin { converged: lock.converged, locked_at_utc: format_iso_utc(lock.locked_at_unix_ms / 1_000), }), + frequency_sweep: self.freq_sweep.as_ref().and_then(|sweep| { + sweep.point().map(|point| FreqSweepSidecar { + min_f: self.min_f, + max_f: self.max_f, + planned_points: self.freq_count as usize, + point_index: sweep.index + 1, + point_total: sweep.points.len(), + order: sweep.order.label().into(), + seed: sweep.seed, + is_reference: point.is_reference, + requested_frequency_hz: point.frequency_hz, + }) + }), pilot: (self.recording.role == RecRole::Pilot) .then_some(self.pilot_windows) .flatten() @@ -2942,6 +3727,9 @@ struct SidecarDoc { /// Present on **event-count** points: the `a₀` lock this point replayed. #[serde(skip_serializing_if = "Option::is_none")] a0_lock: Option, + /// Present on points recorded by the automatic frequency ladder. + #[serde(skip_serializing_if = "Option::is_none")] + frequency_sweep: Option, #[serde(skip_serializing_if = "Option::is_none")] pilot: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -2986,6 +3774,29 @@ struct A0LockSidecar { locked_at_utc: String, } +/// The automatic frequency ladder this point belongs to. +/// +/// The executed order and its seed are part of the frozen session schedule the +/// A1 checklist asks for, so they belong in every point rather than only in an +/// operator's notebook: a block is only interpretable if you can tell which +/// frequency was recorded when. +#[derive(Serialize)] +struct FreqSweepSidecar { + min_f: f64, + max_f: f64, + planned_points: usize, + /// Position in the *executed* order, references included. + point_index: usize, + point_total: usize, + order: String, + seed: u64, + /// True for the interleaved low-frequency reference repeats. + is_reference: bool, + /// The ladder asked for this frequency; `[trigger] measured_frequency_hz` + /// is what the phase-0 markers reported when the point was recorded. + requested_frequency_hz: f64, +} + /// Frozen ON/OFF windows written into a **pilot** recording's sidecar and read /// back to reuse them across the row. #[derive(Serialize, serde::Deserialize)] @@ -3386,8 +4197,11 @@ impl Plugin for StageAA1Plugin { self.scan_measurement_folder(); self.load_a0_locks(); } - // The lock and the sweep run first so a point's recording starts on the - // same tick. They are mutually exclusive, guarded when they begin. + // Outermost first: the frequency sweep starts the lock or the point it + // supervises, and each of those starts its own next stage, so one tick + // carries a hand-off all the way down. They are mutually exclusive at + // the top, guarded where they begin. + self.drive_freq_sweep(context); self.drive_a0_lock(context); self.drive_sweep(context); self.drive_recording(context); @@ -3679,6 +4493,125 @@ impl Plugin for StageAA1Plugin { enabled: can_record, }, }, + SettingItem { + key: "min_f".into(), + label: "Sweep min f (Hz)".into(), + tooltip: Some( + "Lowest frequency of the automatic ladder. It decides whether the \ + ladder is measurable at all: the photodiode needs a contrast \ + window of at least one cycle at this frequency, so raise its \ + Cache length if the sweep refuses to start." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 0.1, + default: self.min_f, + }, + }, + SettingItem { + key: "max_f".into(), + label: "Sweep max f (Hz)".into(), + tooltip: Some( + "Highest frequency of the automatic ladder. Check the refractory \ + condition 2·f·a₀/C ≪ 1/τ_refr here — the plugin does not." + .into(), + ), + kind: SettingKind::F64Drag { + min: 0.01, + max: 2_000.0, + speed: 1.0, + default: self.max_f, + }, + }, + SettingItem { + key: "freq_count".into(), + label: "Frequency points".into(), + tooltip: Some( + "Points on the ladder, log-spaced and inclusive of both ends: \ + |H(f)| is read per decade, so a linear ladder would spend most of \ + its points on the flat part." + .into(), + ), + kind: SettingKind::I64Slider { + min: 1, + max: FREQ_SWEEP_MAX_POINTS as i64, + default: i64::from(self.freq_count), + suffix: None, + }, + }, + SettingItem { + key: "freq_order".into(), + label: "Frequency order".into(), + tooltip: Some( + "Order the ladder is visited in. Low-to-high confounds frequency \ + with anything that drifts through the block (bleaching, thermal \ + bias drift), so prefer alternating or a seeded random order — \ + both are recorded in the sidecar." + .into(), + ), + kind: SettingKind::Enum { + variants: vec![ + "ascending".into(), + "descending".into(), + "alternating".into(), + "random (seeded)".into(), + ], + default: self.freq_order.index() as usize, + }, + }, + SettingItem { + key: "freq_seed".into(), + label: "Random order seed".into(), + tooltip: Some( + "Seed for the random order, so the executed schedule is \ + reproducible and can be frozen in the session plan. Recorded in \ + every point's sidecar." + .into(), + ), + kind: SettingKind::I64Drag { + min: 1, + max: 9_999, + default: self.freq_seed as i64, + }, + }, + SettingItem { + key: "freq_reference_every".into(), + label: "Low-f reference every N points".into(), + tooltip: Some( + "Re-visit the lowest planned frequency after every N points, so \ + drift across the block shows up as a disagreement between its \ + repeats. 0 disables it." + .into(), + ), + kind: SettingKind::I64Slider { + min: 0, + max: 10, + default: i64::from(self.freq_reference_every), + suffix: None, + }, + }, + SettingItem { + key: "start_freq_sweep".into(), + label: "Start frequency sweep (find a₀ + record per f)".into(), + tooltip: Some( + "Runs the whole ladder unattended on one modulation lease: per \ + frequency it retargets the drive, waits for the phase-0 trigger \ + to confirm the new period, locks a₀ closed-loop, and records one \ + atomic RAW + PDQ + sidecar point. A frequency whose a₀ cannot be \ + reached is skipped and named in the summary rather than ending \ + the ladder. The operator's own frequency and depth come back when \ + the lease is released. References (pilot, background, I_tot \ + anchor) and the flux point stay yours — and pilot windows are \ + dropped at every frequency change, because windows frozen at one \ + period do not transfer to another." + .into(), + ), + kind: SettingKind::Button { + enabled: can_record, + }, + }, SettingItem { key: "clear_a0_locks".into(), label: "Clear a₀ lock table".into(), @@ -3824,6 +4757,13 @@ impl Plugin for StageAA1Plugin { "find_a0" => Some(self.press_find_a0.value()), "record_a0_point" => Some(self.press_record_a0.value()), "clear_a0_locks" => Some(self.press_clear_a0.value()), + "min_f" => Some(json!(self.min_f)), + "max_f" => Some(json!(self.max_f)), + "freq_count" => Some(json!(self.freq_count)), + "freq_order" => Some(json!(self.freq_order.index())), + "freq_seed" => Some(json!(self.freq_seed)), + "freq_reference_every" => Some(json!(self.freq_reference_every)), + "start_freq_sweep" => Some(self.press_freq_sweep.value()), // New id regenerates the measurement id locally; the id itself is // what synchronizes, so the press must not be forwarded (both // instances would generate different ids). @@ -3912,9 +4852,16 @@ impl Plugin for StageAA1Plugin { lock.stop_requested = true; self.message = "a₀ lock stop requested".into(); } + // Last, so its wording wins: a stop during a ladder is a + // stop of the ladder, whatever child was mid-flight. + if let Some(sweep) = self.freq_sweep.as_mut() { + sweep.stop_requested = true; + self.message = "Frequency sweep stop requested".into(); + } self.sweep_pending = false; self.a0_lock_pending = false; self.a0_point_pending = false; + self.freq_sweep_pending = false; } } "live" => { @@ -3976,6 +4923,37 @@ impl Plugin for StageAA1Plugin { self.a0_point_pending = true; } } + "min_f" => { + self.min_f = value.as_f64().ok_or("min_f must be a number")?.max(0.01); + } + "max_f" => { + self.max_f = value.as_f64().ok_or("max_f must be a number")?.max(0.01); + } + "freq_count" => { + self.freq_count = value + .as_u64() + .ok_or("freq_count must be an integer")? + .clamp(1, FREQ_SWEEP_MAX_POINTS as u64) + as u32; + } + "freq_order" => { + self.freq_order = + FreqOrder::from_index(value.as_u64().ok_or("freq_order must be an index")?); + } + "freq_seed" => { + self.freq_seed = value.as_u64().ok_or("freq_seed must be an integer")?.max(1); + } + "freq_reference_every" => { + self.freq_reference_every = value + .as_u64() + .ok_or("freq_reference_every must be an integer")? + .min(10) as u32; + } + "start_freq_sweep" => { + if self.press_freq_sweep.accept(&value) { + self.freq_sweep_pending = true; + } + } "clear_a0_locks" => { if self.press_clear_a0.accept(&value) { self.a0_locks.clear(); @@ -4006,6 +4984,29 @@ impl Plugin for StageAA1Plugin { ))); } } + if let Some(sweep) = &self.freq_sweep { + let phase = match sweep.phase { + FreqSweepPhase::AcquiringLease => "leasing modulation", + FreqSweepPhase::SettingFrequency => "retargeting frequency", + FreqSweepPhase::ConfirmingFrequency => "confirming from the trigger", + FreqSweepPhase::Locking => "locking a₀", + FreqSweepPhase::Recording => "recording", + }; + let point = sweep.point(); + entries.push(StatusEntry::Text(format!( + "Frequency sweep {}/{} at {}{} — {phase} ({} recorded, {} skipped)", + sweep.index + 1, + sweep.points.len(), + frequency_label(sweep.frequency_hz()), + if point.is_some_and(|point| point.is_reference) { + " (reference)" + } else { + "" + }, + sweep.recorded, + sweep.failed.len(), + ))); + } if let Some(sweep) = &self.sweep { let phase = match sweep.phase { SweepPhase::AcquiringLease => "leasing modulation", @@ -4299,6 +5300,9 @@ mod tests { for reply in &inbox.service_replies { plugin.on_service_reply(reply); } + // Same order as `process_control`: outermost supervisor first, so one + // tick can carry a hand-off from the ladder down into a recording. + plugin.drive_freq_sweep(sink); plugin.drive_a0_lock(sink); plugin.drive_sweep(sink); plugin.drive_recording(sink); @@ -5008,6 +6012,7 @@ mod tests { lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, lease_req: 0, + owns_lease: true, depth_req: 0, depth_applied: true, settled_since_ms: None, @@ -5079,6 +6084,439 @@ mod tests { let _ = std::fs::remove_dir_all(&folder); } + /// Widens the fixture photodiode's contrast window, so it covers a whole + /// cycle at every frequency a ladder test visits (the lock refuses below + /// one cycle, which is the point of a different test). + fn photodiode_window(plugin: &mut StageAA1Plugin, seconds: f64) { + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(seconds); + } + } + } + + /// Rewrites the plugin's phase-0 markers so the trigger reports `hz`, the + /// way the camera would once the drive has really moved. + fn trigger_reports(plugin: &mut StageAA1Plugin, hz: f64) { + let period_us = (1_000_000.0 / hz).round() as u64; + plugin.camera_markers_us = (0..8).map(|index| index * period_us).collect(); + plugin.fold_cache.replace(None); + } + + /// Drives a whole frequency ladder to completion against a bench that + /// delivers `gain ×` the commanded depth, answering every lease/depth/ + /// frequency request and letting the trigger confirm each commanded + /// frequency. Returns the frequencies whose points were recorded, in order. + fn run_freq_sweep_to_completion( + plugin: &mut StageAA1Plugin, + sink: &mut ControlSink, + gain: f64, + max_ticks: usize, + ) -> Vec { + let mut revision = 1; + let mut recorded = Vec::new(); + control_tick(plugin, PluginControlInbox::default(), sink); + for _ in 0..max_ticks { + let Some((phase, target_hz, lease_req, freq_req, granted, applied)) = + plugin.freq_sweep.as_ref().map(|sweep| { + ( + sweep.phase, + sweep.frequency_hz(), + sweep.lease_req, + sweep.freq_req, + sweep.lease_granted, + sweep.freq_applied, + ) + }) + else { + break; + }; + let mut replies = Vec::new(); + match phase { + FreqSweepPhase::AcquiringLease if !granted => replies.push(accepted(lease_req)), + FreqSweepPhase::SettingFrequency if !applied && freq_req != 0 => { + replies.push(accepted(freq_req)); + } + FreqSweepPhase::ConfirmingFrequency => trigger_reports(plugin, target_hz), + FreqSweepPhase::Locking => { + if let Some(lock) = plugin.a0_lock.as_ref() { + let (depth_req, applied, commanded) = + (lock.depth_req, lock.depth_applied, lock.commanded_a); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = + Some(photodiode_measuring(revision, commanded * gain)); + photodiode_window(plugin, 0.02); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + } + FreqSweepPhase::Recording => { + if let Some(sweep) = plugin.sweep.as_ref() { + let (depth_req, applied, expected) = + (sweep.depth_req, sweep.depth_applied, sweep.target_a()); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, expected)); + photodiode_window(plugin, 0.02); + } + } + // Short-circuit the recording coordinator once the sweep + // has seen the point start: this test is about the ladder, + // and the coordinator has tests of its own. + if plugin.recording.is_active() + && plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started) + { + plugin.recording = Recording::idle(); + plugin.recording_completed_ok = true; + recorded.push(target_hz); + } + } + _ => {} + } + control_tick( + plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + sink, + ); + } + recorded + } + + #[test] + fn the_frequency_ladder_is_log_spaced_and_ordered_reproducibly() { + let mut plugin = plugin_with_markers(); + plugin.min_f = 1.0; + plugin.max_f = 100.0; + plugin.freq_count = 3; + + plugin.freq_order = FreqOrder::Ascending; + let ladder = plugin.planned_frequencies(); + // Log-spaced: |H(f)| is read per decade, so a decade per step. + assert_eq!(ladder.len(), 3); + assert!((ladder[0] - 1.0).abs() < 1e-9); + assert!((ladder[1] - 10.0).abs() < 1e-6, "middle {}", ladder[1]); + assert!((ladder[2] - 100.0).abs() < 1e-6); + + // Alternating decorrelates frequency from time without a seed. + plugin.freq_order = FreqOrder::Alternating; + let order: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + assert!((order[0] - 1.0).abs() < 1e-9 && (order[1] - 100.0).abs() < 1e-6); + assert!((order[2] - 10.0).abs() < 1e-6); + + // A seeded random order is reproducible — the seed is in the sidecar. + plugin.freq_order = FreqOrder::Random; + plugin.freq_count = 8; + plugin.freq_seed = 42; + let first: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + let again: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + assert_eq!(first, again, "the seeded order must be reproducible"); + plugin.freq_seed = 43; + let other: Vec = plugin + .freq_sweep_points() + .iter() + .map(|point| point.frequency_hz) + .collect(); + assert_ne!(first, other, "a different seed must shuffle differently"); + let mut sorted = first.clone(); + sorted.sort_by(f64::total_cmp); + let mut planned = plugin.planned_frequencies(); + planned.sort_by(f64::total_cmp); + assert_eq!(sorted.len(), planned.len(), "the shuffle is a permutation"); + } + + #[test] + fn the_low_frequency_reference_is_interleaved_into_the_ladder() { + let mut plugin = plugin_with_markers(); + plugin.min_f = 1.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 4; + plugin.freq_order = FreqOrder::Ascending; + plugin.freq_reference_every = 2; + + let points = plugin.freq_sweep_points(); + let flags: Vec = points.iter().map(|point| point.is_reference).collect(); + assert_eq!(flags, [false, false, true, false, false, true]); + for point in points.iter().filter(|point| point.is_reference) { + assert!( + (point.frequency_hz - 1.0).abs() < 1e-9, + "the reference repeats the lowest planned frequency" + ); + } + } + + #[test] + fn the_frequency_sweep_locks_and_records_every_point_on_one_lease() { + let folder = temp_folder("fsweep"); + let mut plugin = plugin_locking(0.6, &folder); + plugin.a0_target = 0.5; + plugin.min_f = 100.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 2; + plugin.freq_order = FreqOrder::Ascending; + photodiode_window(&mut plugin, 0.02); + plugin.freq_sweep_pending = true; + let mut sink = ControlSink::default(); + + let recorded = run_freq_sweep_to_completion(&mut plugin, &mut sink, 0.6, 4_000); + + assert_eq!(recorded.len(), 2, "message: {}", plugin.message); + assert!((recorded[0] - 100.0).abs() < 1.0 && (recorded[1] - 1_000.0).abs() < 10.0); + assert!(plugin.freq_sweep.is_none(), "the ladder must finish"); + assert!( + plugin.message.contains("2/2 points recorded"), + "message: {}", + plugin.message + ); + + // One lease for the whole ladder: the operator's drive settings are + // locked out from the first frequency to the last, so the amplitude + // provably cannot move between a lock and the point that replays it. + let commands: Vec = sink + .services + .iter() + .filter_map(|request| { + serde_json::from_value::(request.payload.clone()) + .ok() + .map(|envelope| envelope.command) + }) + .collect(); + let acquired = commands + .iter() + .filter(|command| matches!(command, ModulationCommandV1::AcquireLease { .. })) + .count(); + let released = commands + .iter() + .filter(|command| matches!(command, ModulationCommandV1::ReleaseLease { .. })) + .count(); + assert_eq!(acquired, 1, "one lease for the ladder, not one per child"); + assert_eq!(released, 1, "released exactly once, at the end"); + assert!(commands + .iter() + .any(|command| matches!(command, ModulationCommandV1::SetDriveFrequency { .. }))); + + // Both frequencies are locked, each at the depth its own roll-off needs. + assert_eq!(plugin.a0_locks.len(), 2); + for lock in &plugin.a0_locks { + assert!( + lock.converged, + "lock at {} did not converge", + lock.frequency_hz + ); + assert!((lock.measured_a - 0.5).abs() <= plugin.a0_tolerance); + } + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn a_frequency_the_trigger_never_confirms_is_skipped_not_fatal() { + // The firmware ACKs a table it accepted, not light that is modulating. + // A point whose trigger never reports the commanded period is skipped + // and named; the rest of the ladder is still worth having. + let folder = temp_folder("fskip"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.min_f = 100.0; + plugin.max_f = 1_000.0; + plugin.freq_count = 2; + plugin.freq_order = FreqOrder::Ascending; + photodiode_window(&mut plugin, 0.02); + plugin.freq_sweep_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + let mut revision = 1; + let mut recorded = Vec::new(); + for _ in 0..4_000 { + let Some((phase, target_hz, lease_req, freq_req, granted, applied)) = + plugin.freq_sweep.as_ref().map(|sweep| { + ( + sweep.phase, + sweep.frequency_hz(), + sweep.lease_req, + sweep.freq_req, + sweep.lease_granted, + sweep.freq_applied, + ) + }) + else { + break; + }; + let mut replies = Vec::new(); + match phase { + FreqSweepPhase::AcquiringLease if !granted => replies.push(accepted(lease_req)), + FreqSweepPhase::SettingFrequency if !applied && freq_req != 0 => { + replies.push(accepted(freq_req)); + } + FreqSweepPhase::ConfirmingFrequency => { + // The trigger confirms 100 Hz but never moves to 1 kHz. + if target_hz < 500.0 { + trigger_reports(&mut plugin, target_hz); + } else if let Some(sweep) = plugin.freq_sweep.as_mut() { + sweep.confirm_deadline_ms = 1; + } + } + FreqSweepPhase::Locking => { + if let Some(lock) = plugin.a0_lock.as_ref() { + let (depth_req, applied, commanded) = + (lock.depth_req, lock.depth_applied, lock.commanded_a); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, commanded)); + photodiode_window(&mut plugin, 0.02); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + } + FreqSweepPhase::Recording => { + if let Some(sweep) = plugin.sweep.as_ref() { + let (depth_req, applied, expected) = + (sweep.depth_req, sweep.depth_applied, sweep.target_a()); + if !applied && depth_req != 0 { + replies.push(accepted(depth_req)); + } else { + revision += 1; + plugin.photodiode = Some(photodiode_measuring(revision, expected)); + photodiode_window(&mut plugin, 0.02); + } + } + if plugin.recording.is_active() + && plugin + .sweep + .as_ref() + .is_some_and(|sweep| sweep.point_started) + { + plugin.recording = Recording::idle(); + plugin.recording_completed_ok = true; + recorded.push(target_hz); + } + } + _ => {} + } + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: replies, + ..PluginControlInbox::default() + }, + &mut sink, + ); + } + + assert_eq!(recorded.len(), 1, "message: {}", plugin.message); + assert!(plugin.freq_sweep.is_none()); + assert!( + plugin.message.contains("1/2 points recorded") && plugin.message.contains("1 skipped"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn the_frequency_sweep_refuses_a_ladder_its_photodiode_cannot_measure() { + // The estimator window is one window for the whole ladder, so the + // *lowest* point decides measurability. Refuse the plan, not its + // ninth point two hours in. + let folder = temp_folder("fladder"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.a0_target = 0.5; + plugin.min_f = 0.1; + plugin.max_f = 100.0; + plugin.freq_count = 4; + if let Some(summary) = plugin.photodiode.as_mut() { + if let Some(optical) = summary.optical_summary.as_mut() { + optical.window_seconds = Some(1.0); // 0.1 cycles at 0.1 Hz + } + } + plugin.freq_sweep_pending = true; + let mut sink = ControlSink::default(); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert!(plugin.freq_sweep.is_none(), "the ladder must not start"); + assert!(sink.services.is_empty(), "no lease may be requested"); + assert!( + plugin.message.contains("lowest point") && plugin.message.contains("cache length"), + "message: {}", + plugin.message + ); + let _ = std::fs::remove_dir_all(&folder); + } + + #[test] + fn changing_frequency_drops_the_previous_period_s_markers_and_windows() { + // The measured period is the mean marker spacing, so markers from the + // old drive would confirm the new frequency against a mixture. Pilot + // windows are frozen at a phase of the old period and do not transfer. + let folder = temp_folder("fflush"); + let mut plugin = plugin_locking(1.0, &folder); + plugin.pilot_windows = Some(( + PhaseWindow { + start: 0.0, + end: 0.2, + }, + PhaseWindow { + start: 0.5, + end: 0.7, + }, + )); + plugin.freq_sweep = Some(FreqSweep { + phase: FreqSweepPhase::AcquiringLease, + points: vec![FreqSweepPoint { + frequency_hz: 50.0, + is_reference: false, + }], + index: 0, + lease_id: LeaseId::new("a1-fsweep-test"), + lease_granted: true, + lease_req: 0, + freq_req: 0, + freq_applied: false, + confirm_deadline_ms: 0, + skip_reason: None, + failed: Vec::new(), + recorded: 0, + order: FreqOrder::Ascending, + seed: 1, + last_activity_ms: now_unix_ms(), + stop_requested: false, + }); + let mut sink = ControlSink::default(); + plugin.send_freq_sweep_frequency(&mut sink); + + assert!(plugin.camera_markers_us.is_empty()); + assert!(plugin.camera_events.is_empty()); + assert!( + plugin.pilot_windows.is_none(), + "windows frozen at another period must not carry over" + ); + let _ = std::fs::remove_dir_all(&folder); + } + #[test] fn a0_lock_refuses_a_photodiode_window_shorter_than_one_cycle() { // `a` is peak-to-peak. Under one cycle the photodiode under-reports it, @@ -5303,6 +6741,7 @@ mod tests { lease_id: LeaseId::new("a1-sweep-test"), lease_granted: true, lease_req: 0, + owns_lease: true, depth_req: 0, depth_applied: true, settled_since_ms: None, diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index f60ac8f..563b655 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -909,6 +909,9 @@ pub struct StageAModulationPlugin { /// The operator's armed `depth_a`, parked while a lease drives the optical /// depth (A1's amplitude sweep) and restored by [`Self::end_lease`]. armed_depth_a: Option, + /// The operator's armed `frequency_hz`, parked while a lease drives the + /// frequency (A1's frequency sweep) and restored by [`Self::end_lease`]. + armed_frequency_hz: Option, /// Operating illumination `I_k` as a normalised lobe intensity `u_k ∈ (0,1]`. /// Held fixed while `a` is swept, so one response curve keeps `I_k` constant. operating_point: f64, @@ -983,6 +986,7 @@ impl Default for StageAModulationPlugin { frequency_hz: 10.0, depth_a: 0.5, armed_depth_a: None, + armed_frequency_hz: None, operating_point: 0.5, v_null_dac: 0, v_pi_dac: 2_048, @@ -2004,6 +2008,63 @@ impl StageAModulationPlugin { self.shared.bump(); self.immediate_response(request, RequestOutcomeV1::Applied, None) } + ModulationCommandV1::SetDriveFrequency { frequency_millihz } => { + self.require_lease(request)?; + if self.link.is_none() { + return Err(service_error( + ServiceErrorCodeV1::NotConnected, + "the modulation owner is not connected to the device", + false, + )); + } + let frequency_hz = *frequency_millihz as f64 / 1_000.0; + // The same band `drive_command` clamps to; refuse rather than + // silently record a different frequency than the one asked for. + if !(0.01..=2_000.0).contains(&frequency_hz) { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + format!( + "frequency {frequency_hz:.3} Hz outside the supported \ + 0.01..=2000 Hz" + ), + false, + )); + } + // A constant hold has no frequency, and the manual DAC band is + // not the calibrated drive this path retargets. + if self.method == DriveMethod::Manual || self.mode == Mode::Const { + return Err(service_error( + ServiceErrorCodeV1::InvalidCommand, + "arm a calibrated periodic/optical drive in the modulation plugin \ + before sweeping the frequency", + false, + )); + } + let previous = self.frequency_hz; + self.frequency_hz = frequency_hz; + let command = match self.drive_command() { + Ok(command) => command, + Err(error) => { + self.frequency_hz = previous; + return Err(service_error( + ServiceErrorCodeV1::DeviceRejected, + format!("frequency {frequency_hz:.3} Hz rejected: {error}"), + false, + )); + } + }; + // As for the depth: park the operator's own frequency on the + // first retarget only, so `end_lease` hands back what they + // armed rather than the sweep's last point. + self.armed_frequency_hz.get_or_insert(previous); + *self.shared.pending.lock().expect("pending lock") = Some(PendingOperation { + commands: vec![command], + purpose: "MOD", + meta: None, + }); + self.shared.bump(); + self.immediate_response(request, RequestOutcomeV1::Applied, None) + } ModulationCommandV1::PrepareA1 { configuration } => { self.require_lease(request)?; let revision = self.requested_revision(request)?; @@ -2160,8 +2221,15 @@ impl StageAModulationPlugin { /// restores through `Sweep::restore`; this is the leased equivalent. fn end_lease(&mut self) { self.lease = None; - if let Some(depth) = self.armed_depth_a.take() { + let depth = self.armed_depth_a.take(); + let frequency = self.armed_frequency_hz.take(); + if let Some(depth) = depth { self.depth_a = depth; + } + if let Some(frequency) = frequency { + self.frequency_hz = frequency; + } + if depth.is_some() || frequency.is_some() { // Re-arm the board only if nobody else now owns the DAC; // `send_modulation` is itself guarded. self.send_modulation(); diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 545644b..29b2b33 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -293,6 +293,19 @@ pub enum ModulationCommandV1 { SetOpticalDepth { depth_a_milli: u32, }, + /// Retarget the armed drive's *frequency*, leaving everything else — the + /// waveform shape, the depth, the operating point and the calibration — as + /// the operator armed it. The frequency counterpart of + /// [`ModulationCommandV1::SetOpticalDepth`], and the same scoping rules + /// apply: leased only, rejected when the link is closed or the armed drive + /// has no frequency to retarget (manual DAC method, constant mode). + /// + /// A1's frequency sweep drives this. The owner parks the operator's armed + /// frequency on the first one and restores it when the lease ends, so a + /// finished sweep does not leave the bench on its last point. + SetDriveFrequency { + frequency_millihz: u64, + }, PrepareA1 { configuration: A1AcquisitionConfigV1, }, From 5011dac9c1951fa282c5a1ab4491c4df300da8e8 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Sat, 25 Jul 2026 17:22:50 +0200 Subject: [PATCH 28/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20keep=20A1?= =?UTF-8?q?=20recordings=20full-length=20and=20in=20one=20folder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starting an A1 recording appeared to succeed and then reported a finished run immediately. What landed was a config sidecar in the chosen output folder, a truncated camera .raw in the host process's working directory, and no .pdq at all. Three defects combined: - Every photodiode-leg failure jumped straight to stop_camera, so the host had been recording for a few hundred milliseconds and produced a stub RAW that still carried a complete finalization receipt. Photodiode faults also set stop_requested, conflating them with an operator stop. The camera now runs its full duration and closes as a camera-only recording instead. - The specific cause ('set the data directory first') was overwritten by the generic 'was incomplete' on the way out. The first cause is now preserved and named in the closing message. - The RAW, the PDQ, and the sidecar are written by three owners against three roots, and the host's relative output path resolved to its working directory. Once both recorders report finalization their files are closed and hashed, so A1 now gathers them into // and records the final paths. PDQ receipts report a label relative to the photodiode data directory, which the owner now publishes in its summary so the path can be resolved. Also pre-flights the photodiode before starting the camera (not reporting, not connected, no data directory, leased elsewhere), surfaces the blocker in the status view while idle, and stops A1's own pipeline restart from wiping the row's pilot windows, background floor, and collected response points. Cherry-picked from fix/stage-a-a1-recording onto this branch, because the frequency ladder records every one of its points through this coordinator: without these fixes an unattended ladder would write a folder of truncated RAWs and no PDQ at all. Its ADR is renumbered 012 to 015 — the third branch to have claimed 012 independently. Refs ADR 015, revises ADR 009 decision 3. --- .../009-stage-a-a1-recording-coordinator.md | 10 +- .../015-stage-a-a1-recording-robustness.md | 112 ++++ docs/features/stage-a-a1.md | 66 +- plugins/stage-a-a1/src/runtime.rs | 567 +++++++++++++++++- plugins/stage-a-photodiode/src/lib.rs | 4 + stage-a-plugin-contract/src/lib.rs | 5 + 6 files changed, 713 insertions(+), 51 deletions(-) create mode 100644 docs/adr/015-stage-a-a1-recording-robustness.md diff --git a/docs/adr/009-stage-a-a1-recording-coordinator.md b/docs/adr/009-stage-a-a1-recording-coordinator.md index c56c977..3b446f1 100644 --- a/docs/adr/009-stage-a-a1-recording-coordinator.md +++ b/docs/adr/009-stage-a-a1-recording-coordinator.md @@ -1,6 +1,8 @@ # ADR 009 — Stage-A A1 as a focused recording coordinator -- **Status:** Accepted +- **Status:** Accepted — decision 3 revised by + [ADR 015](015-stage-a-a1-recording-robustness.md), which makes A1's output + folder authoritative and gathers the RAW/PDQ into it after finalization - **Date:** 2026-07-23 - **Relates to:** ADR 005 (device ownership), ADR 006 (two-plugin split), ADR 007 (owner orchestration — the earlier, broader orchestrator), @@ -100,8 +102,10 @@ phase-anchoring knobs (event latency, self-align) that the now-reliable not a rolling internal log. - The host returns to Preview before delivering its final receipt, which keeps repeated recordings and automated sweeps live without an extra operator step. -- True single-directory co-location is a **configuration** convention (align the +- ~~True single-directory co-location is a **configuration** convention (align the recorder roots), not something A1 enforces. Enforcing it would require host and - photodiode path changes and is out of scope. + photodiode path changes and is out of scope.~~ **Revised by ADR 015:** A1 moves + the finalized files into its own measurement folder, which needs no host or + photodiode path changes because both files are already closed and hashed. - The contract and ABI are unchanged: every message used already exists (`HostCommand`, `PhotodiodeCommandV1` lease/begin/finalize). diff --git a/docs/adr/015-stage-a-a1-recording-robustness.md b/docs/adr/015-stage-a-a1-recording-robustness.md new file mode 100644 index 0000000..15af6f3 --- /dev/null +++ b/docs/adr/015-stage-a-a1-recording-robustness.md @@ -0,0 +1,112 @@ +# ADR 015 — Stage-A A1 recording: one folder, full duration, named failures + +- **Status:** Accepted +- **Date:** 2026-07-25 +- **Relates to:** ADR 009 (A1 as a recording coordinator — revises decision 3 and + its co-location consequence), ADR 005 (device ownership), + ADR 006 (two-plugin split), + [Stage-A A1 Analysis](../features/stage-a-a1.md) + +## Context + +On the bench, *Start recording* looked like it worked and then reported a +finished run almost immediately. What actually landed on disk was: + +- an A1 `_config.toml` in the chosen output folder, +- a **truncated** camera `.raw` (plus the host's bias `.toml`) in an unrelated + directory — the host process's working directory, +- **no `.pdq` and no photodiode sidecar at all**, +- and a status message that said only "was incomplete". + +Three separate defects produced that outcome. + +1. **A photodiode failure cut the camera recording short.** The coordinator + starts the camera first, then connects, leases, and opens the PDQ. Every + photodiode-leg failure — a rejected `Connect`, a refused lease, or a + `BeginRecording` rejected because the photodiode's *Data directory* was unset + — jumped straight to `stop_camera`. The host had been recording for a few + hundred milliseconds, so the RAW was a stub that nevertheless carried a + complete finalization receipt. `stop_requested` was also set on photodiode + faults, conflating "the operator asked to stop" with "the photodiode broke". + +2. **The failure reason was discarded.** Each failure wrote a specific message + (`Photodiode start failed (invalid_path): set the data directory first`), and + `finish_recording` then overwrote it with the generic "was incomplete". + The one piece of information the operator needed was destroyed on the way out. + +3. **One measurement scattered across up to three roots.** Per ADR 009 decision 3 + each recorder confines its own writes: the host resolves plugin recording paths + below *its* output directory and **rejects absolute paths**; the photodiode + resolves PDQ paths below *its* data directory; A1 writes its sidecar below + *its* output folder. ADR 009 accepted this and called physical co-location a + configuration convention. In practice the host's output path was relative, so + its parent resolved to the process working directory, and the RAW landed in a + source checkout — nowhere near the experiment folder. + +## Decision + +1. **The camera RAW always runs its full duration.** A photodiode failure while + the camera is already recording no longer stops it. The run continues to the + requested duration and closes normally, with the sidecar and message marking + it camera-only. A complete camera-only recording is a usable measurement; a + truncated file that reports itself as finalized is a trap. `stop_requested` + now means only what its name says — an operator stop — and photodiode faults + travel in `pd_rejected`. + +2. **The photodiode is pre-flighted before the camera starts.** A recording is + refused, with nothing recorded and an actionable message, when the photodiode + is not reporting status, is not connected, has no data directory, or is leased + by another client. These were exactly the conditions that used to surface as a + PDQ rejection *after* the host was already recording. The same check feeds the + A1 status view while idle, so the blocker is visible **before** the operator + presses Record rather than after a wasted run. + + This needs the owner's data directory, so `PhotodiodeSummaryV1` gains an + additive `data_dir: Option` field (`#[serde(default)]`, absent from + older owners, ignored by older consumers — the contract version is unchanged). + +3. **The first failure is preserved and named.** `Recording::failure` keeps the + first, most specific cause; later fallout cannot overwrite it. The closing + message reads `Recording incomplete: — metadata saved to `. + +4. **A1's output folder becomes authoritative for the whole measurement.** After + both recorders report finalization, A1 moves the RAW, the host's bias sidecar, + and the PDQ and its sidecar into `//`, then writes the + config sidecar with the final paths. This reverses ADR 009's "co-location is a + configuration convention" without touching the host or photodiode path rules: + both files are closed and hashed by the time their receipts arrive, so moving + them afterwards is safe and stays inside each owner's contract. + + The move is a `rename` on one volume and a size-verified copy-then-delete + across volumes. It never overwrites an existing destination and never removes + a source it has not verified; if a move fails, the file stays put and the + sidecar records where it actually is. + + PDQ receipts report the path **label** A1 requested — relative to the + photodiode's data directory — not an absolute path, so A1 resolves it against + the `data_dir` from decision 2 before locating the file. The sidecar records + the resolved absolute path either way, which also fixes the previous ambiguity + of storing a bare relative label under `[files]`. + +5. **Self-inflicted pipeline restarts no longer wipe the row.** Starting and + stopping the host recorder restarts the capture pipeline, which the host + reports as `SourceChanged` — twice per recording, caused by A1 itself. That + used to clear the pilot windows, the background floor, and every response + point collected across a sweep. While a recording or sweep is in flight the + boundary now resets only the event fold, whose timeline genuinely did restart. + +## Consequences + +- A recording can now end as *camera-only*: `recording_completed_ok` stays false, + so an amplitude sweep still stops rather than silently collecting points with + no measured `a`. The RAW is complete and reusable. +- A misconfigured bench refuses to record instead of producing a stub. This is a + deliberate behaviour change: pressing Record with a disconnected photodiode + now yields a message and no files, where it previously yielded a junk RAW. +- The measurement folder is the single place to look. Files are no longer where + the host and photodiode settings happen to point, so operators do not have to + keep three roots aligned by hand. Aligning them is still harmless — a file + already in the destination is left alone. +- Moving a large RAW across volumes copies it. On one volume (the normal case) + the move is a metadata operation regardless of file size. +- The contract addition is additive and backward compatible; no ABI change. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index b230c53..8b0f152 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -76,16 +76,21 @@ the drive to still be where a previous action left it (ADR 013). **Naming.** Files share an `_[_role]` stem under an `/` subfolder (`_pilot` / `_background` tag the reference runs, `_ec_fHz` an event-count point): -- `/_.raw` — camera RAW, under the **host output root**, with the host's - own `.toml` sidecar (camera biases, ROI) written next to it. -- `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar, under the - **photodiode data root**. -- `/__config.toml` — the A1 sidecar, under the chosen output folder. - -Each recorder confines its writes to its own root, so A1 cannot force one absolute -directory (see ADR 009). Point the host output root and the photodiode data root -at the same experiment directory to co-locate everything; the A1 sidecar records -the *resolved* paths so the set stays linked either way. +- `/_.raw` — camera RAW, with the host's own `.toml` sidecar + (camera biases, ROI) next to it. +- `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar. +- `/__config.toml` — the A1 sidecar. + +**Everything lands under `//`.** The two recorders each +write below their own root while recording — the host resolves plugin recording +paths below *its* output directory and rejects absolute ones, the photodiode +resolves PDQ paths below *its* data directory — so once both files are finalized +(closed and hashed) A1 moves them into the measurement folder and records the +final paths in the sidecar (ADR 015). The A1 output folder is therefore the only +setting that decides where a measurement ends up; the host and photodiode roots +no longer have to be kept aligned by hand. A move is a rename on one volume and a +size-verified copy across volumes; a file that cannot be moved stays where it is +and the sidecar points at it there. **A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the @@ -111,15 +116,32 @@ one concise result or error message; it does not render an internal event log. A1 declares `host_commands = ["start_recording", "stop_recording"]` in its manifest. Every role uses this same lifecycle. +**When something is wrong** (ADR 015): + +- **Before the camera starts**, A1 refuses the recording — writing nothing — if + the photodiode is not reporting status, not connected, has no data directory, + or is leased by someone else. The same hint fills the status `message` cell + while idle, so it is visible before the button is pressed. +- **If the photodiode fails once the camera is running**, the camera keeps + recording for the full requested duration and closes normally. The run is + marked camera-only: `recording_completed_ok` stays false (so a sweep stops), + but the RAW is complete rather than a truncated stub. +- **The first, most specific failure is what you see.** The closing message is + `Recording incomplete: — metadata saved to `; later fallout + cannot overwrite the original cause. +- **Starting and stopping the host recorder restarts the capture pipeline**, which + the host reports as a `SourceChanged` discontinuity — twice per recording. While + a recording or sweep is in flight that boundary resets only the event fold, not + the row's pilot windows, background floor, or collected response points. + **Host-side note.** The camera RAW leg restarts the host pipeline into Recording mode and stops it again at finalize. After the file is finalized, the host restores Preview before returning the receipt, so a sweep or another button press can start the next recording automatically. -**File locations** (three roots, point them at the same experiment directory): -`//.raw` (+ host `.toml`), -`//_pd.pdq` + `_pd.json`, and -`//_config.toml`. +**File locations.** One place: `//` holds `.raw` +(+ the host's `.toml`), `_pd.pdq` + `_pd.json`, and +`_config.toml`. ## The two live plots @@ -193,8 +215,9 @@ used to do nothing — the presses died on the mirror. Related: A1 overrides `on_discontinuity` to ignore `SettingsChanged` (raised on *every* settings sync of any plugin), so the response curve, pilot windows and -background floor survive ordinary UI interaction; source changes and seeks -still reset everything. +background floor survive ordinary UI interaction. Source changes and seeks reset +everything **unless** a recording or sweep is in flight, in which case the +boundary is A1's own pipeline restart and only the event fold resets (ADR 015). ## Where the inputs come from @@ -215,5 +238,12 @@ the pilot-window round-trip through the measurement folder, press-latch edge/bas semantics, the jittery-marker free-running fallback, sweep-point spacing, the sweep-point sidecar fields, the ordered camera → PDQ → PDQ finalize → camera finalize lifecycle (including envelope identity/revision and save location), the -selective discontinuity reset, and the `a₀`-lock set listed in the -[exact-event-count brief](./stage-a-a1-event-count.md). +selective discontinuity reset, and the `a₀`-lock and frequency-ladder sets listed +in the [exact-event-count brief](./stage-a-a1-event-count.md). + +Three of them guard the recording defects fixed in ADR 015: a photodiode leg that +cannot start is refused before any host command is sent; a photodiode failure +mid-run keeps the camera recording for the full duration, names the cause in the +closing message, and still gathers the RAW and its bias sidecar into the +measurement folder; and a self-inflicted `SourceChanged` during a recording keeps +the row's response points and pilot windows while still resetting the event fold. diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 209bdb9..5133f6f 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -332,6 +332,9 @@ struct Recording { /// The photodiode rejected BeginRecording — skip the finalize and don't /// wait for its receipt. pd_rejected: bool, + /// First thing that went wrong, kept verbatim so the closing message names + /// the cause instead of only reporting that the run was incomplete. + failure: Option, } /// Where the amplitude sweep is within its per-point cycle. @@ -835,6 +838,15 @@ impl Recording { pd_finalized: false, pd_valid: false, pd_rejected: false, + failure: None, + } + } + + /// Records the first failure only: later fallout (a stop that finds nothing + /// to finalize) must not mask the reason the run went wrong. + fn fail(&mut self, reason: impl Into) { + if self.failure.is_none() { + self.failure = Some(reason.into()); } } @@ -893,6 +905,18 @@ impl StageAA1Plugin { self.bump(); } + /// Reports a failure and remembers it as the run's cause, so the closing + /// message can name it after the coordinator has unwound. The first cause + /// wins: it is the specific one, and later arms only see the fallout. + fn note_failure(&mut self, message: impl Into) { + if self.recording.failure.is_some() { + return; + } + let message = message.into(); + self.recording.fail(message.clone()); + self.note(message); + } + /// The modulation period `T` in microseconds: measured from the phase-0 /// markers when present (the trigger *defines* the frequency, latency- /// invariant), otherwise the modulation plugin's acknowledged waveform. @@ -1296,10 +1320,13 @@ impl StageAA1Plugin { cell("events", self.camera_events.len().to_string()), cell( "message", - if self.message.is_empty() { - "—".into() - } else { - self.message.clone() + // While idle, anything that would refuse the next recording + // is worth more than the previous run's result: the operator + // sees it before pressing Record, not after. + match (self.recording.is_active(), self.photodiode_blocker()) { + (false, Some(blocker)) => blocker, + _ if self.message.is_empty() => "—".into(), + _ => self.message.clone(), }, ), ], @@ -1439,6 +1466,42 @@ impl StageAA1Plugin { meta } + /// Why the photodiode cannot record right now, phrased as the operator + /// action that fixes it. `None` means the PDQ leg is expected to succeed. + fn photodiode_blocker(&self) -> Option { + let Some(photodiode) = self.photodiode.as_ref() else { + return Some( + "The photodiode plugin is not reporting status — enable it before recording".into(), + ); + }; + if !matches!(photodiode.connection, ConnectionStateV1::Connected { .. }) { + return Some(format!( + "The photodiode is {} — connect it before recording", + connection_label(&photodiode.connection) + )); + } + if photodiode + .data_dir + .as_ref() + .is_none_or(|folder| folder.trim().is_empty()) + { + return Some( + "Set the photodiode Data directory before recording — the PDQ has nowhere to go" + .into(), + ); + } + // A lease held by anyone else means the PDQ is already committed. + if let Some(lease) = photodiode.lease.as_ref() { + if lease.holder.as_str() != A1_PLUGIN_ID { + return Some(format!( + "The photodiode is leased by {} — release it before recording", + lease.holder.as_str() + )); + } + } + None + } + /// Kick off a coordinated recording by starting the camera first. Called /// on the control tick after a record button is pressed. fn begin_recording(&mut self, context: &mut impl RecordingControl, role: RecRole) { @@ -1453,6 +1516,13 @@ impl StageAA1Plugin { self.note("Set a measurement id before recording"); return; } + // Checked before the camera starts: every one of these used to surface + // as a PDQ rejection *after* the host was already recording, which left + // a stub RAW behind and no photodiode data. + if let Some(blocker) = self.photodiode_blocker() { + self.note(blocker); + return; + } let now_ms = now_unix_ms(); let id = sanitize_stem(self.measurement_id.trim()); // Sweep points get a stable per-point tag so the row's files sort by @@ -1630,6 +1700,31 @@ impl StageAA1Plugin { )); } + /// The photodiode leg failed while the camera was already recording. The + /// camera RAW is the primary measurement, so it keeps running for its full + /// duration instead of being cut short — a truncated file that reports + /// itself as finalized is worse than a complete camera-only one. Any lease + /// still held is released by the normal stop path at the end. + fn continue_without_photodiode(&mut self, context: &mut impl RecordingControl) { + let camera_running = self.recording.cam_raw_path.is_some() && !self.recording.cam_rejected; + if !camera_running || self.recording.stop_requested { + self.stop_camera(context); + return; + } + self.recording.phase = RecPhase::Running; + self.recording.start_unix_ms = now_unix_ms(); + self.recording.last_activity_ms = self.recording.start_unix_ms; + let reason = self + .recording + .failure + .clone() + .unwrap_or_else(|| "the photodiode did not start".into()); + self.note(format!( + "{reason} — recording camera only for {} s", + self.recording.duration_s + )); + } + /// Stop the host recorder after the PDQ has been safely finalized. fn stop_camera(&mut self, context: &mut impl RecordingControl) { if self.recording.cam_raw_path.is_some() && !self.recording.cam_rejected { @@ -1656,11 +1751,19 @@ impl StageAA1Plugin { && self.recording.pd_valid && self.recording.pd_pdq_path.is_some() && self.recording.pd_sidecar_path.is_some(); + // Gather the RAW/PDQ next to the sidecar before writing it, so the + // recorded paths are the final ones. + self.gather_into_measurement_folder(); let sidecar = self.write_sidecar(); + let reason = self + .recording + .failure + .clone() + .unwrap_or_else(|| "not every file was finalized".into()); let message = match (sidecar, clean) { (Ok(path), true) => format!("Saved recording {} → {path}", self.recording.id), (Ok(path), false) => format!( - "Recording {} was incomplete — metadata saved to {path}", + "Recording {} incomplete: {reason} — metadata saved to {path}", self.recording.id ), (Err(err), _) => format!( @@ -1672,6 +1775,67 @@ impl StageAA1Plugin { self.release_and_idle(context, message); } + /// Collects the finalized artifacts into `//`. + /// + /// The camera RAW and the PDQ are written by two other owners against their + /// own roots — the host resolves plugin recording paths below *its* output + /// directory and rejects absolute ones, and the photodiode resolves PDQ + /// paths below *its* data directory. Left alone, one measurement scatters + /// across up to three unrelated folders. Both files are closed and hashed + /// by the time their receipts arrive, so moving them here is safe and makes + /// this plugin's output folder authoritative for the whole measurement. + fn gather_into_measurement_folder(&mut self) { + let dir = PathBuf::from(&self.recording.folder).join(&self.recording.id); + if std::fs::create_dir_all(&dir).is_err() { + return; + } + let raw = self + .recording + .cam_finalized_path + .clone() + .or_else(|| self.recording.cam_raw_path.clone()); + // The host writes the bias/config sidecar as a sibling of the RAW; it + // travels with it so the recording stays self-describing. + if let Some(raw) = raw { + if let Some(moved) = move_into(&dir, &raw) { + if self.recording.cam_finalized_path.is_some() { + self.recording.cam_finalized_path = Some(moved.clone()); + } + self.recording.cam_raw_path = Some(moved); + } + if let Some(bias) = sibling_toml(&raw) { + move_into(&dir, &bias); + } + } + // PDQ receipts report the *label* A1 asked for, which is relative to the + // photodiode's data directory — resolve it before touching the file, and + // record the absolute path either way. + if let Some(pdq) = self.resolved_photodiode_path(self.recording.pd_pdq_path.as_deref()) { + self.recording.pd_pdq_path = Some(move_into(&dir, &pdq).unwrap_or(pdq)); + } + if let Some(sidecar) = + self.resolved_photodiode_path(self.recording.pd_sidecar_path.as_deref()) + { + self.recording.pd_sidecar_path = Some(move_into(&dir, &sidecar).unwrap_or(sidecar)); + } + } + + /// Absolute location of a photodiode-reported recording path. The owner + /// reports paths relative to its own data directory, which it publishes in + /// its summary; an already-absolute path is taken as given. + fn resolved_photodiode_path(&self, reported: Option<&str>) -> Option { + let reported = reported?; + let path = Path::new(reported); + if path.is_absolute() { + return Some(reported.to_owned()); + } + let root = self + .photodiode + .as_ref() + .and_then(|photodiode| photodiode.data_dir.as_deref())?; + Some(Path::new(root).join(path).display().to_string()) + } + /// Release the photodiode lease (only if we actually hold it) and return to idle. fn release_and_idle(&mut self, context: &mut impl RecordingControl, message: String) { if self.recording.lease_granted { @@ -3385,7 +3549,7 @@ impl StageAA1Plugin { HostCommandOutcome::Rejected { code, message } => { // Stop the rest of the recording; drive_recording resolves the // abort from the current phase on the next tick. - self.note(format!("Camera recording rejected ({code}): {message}")); + self.note_failure(format!("Camera recording rejected ({code}): {message}")); self.recording.cam_rejected = true; self.recording.stop_requested = true; } @@ -3407,7 +3571,7 @@ impl StageAA1Plugin { self.recording.last_activity_ms = now_unix_ms(); } HostCommandOutcome::Rejected { code, message } => { - self.message = format!("Camera stop failed ({code}): {message}"); + self.note_failure(format!("Camera stop failed ({code}): {message}")); self.recording.cam_rejected = true; self.recording.last_activity_ms = now_unix_ms(); } @@ -3432,11 +3596,13 @@ impl StageAA1Plugin { || reply.request_id == self.recording.lease_req || reply.request_id == self.recording.pd_begin_req { - self.note(format!("Photodiode start failed ({code}): {message}")); + // Not `stop_requested`: that flag means the operator asked + // to stop. A photodiode fault leaves the camera running to + // its full duration (see `continue_without_photodiode`). + self.note_failure(format!("Photodiode start failed ({code}): {message}")); self.recording.pd_rejected = true; - self.recording.stop_requested = true; } else if reply.request_id == self.recording.pd_finalize_req { - self.note(format!("Photodiode save failed ({code}): {message}")); + self.note_failure(format!("Photodiode save failed ({code}): {message}")); self.recording.pd_rejected = true; self.recording.lease_granted = false; self.recording.last_activity_ms = now_unix_ms(); @@ -3493,11 +3659,15 @@ impl StageAA1Plugin { } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.cam_rejected = true; - self.release_and_idle(context, "Timed out starting camera recording".into()); + self.note_failure("Timed out starting camera recording"); + let message = self.message.clone(); + self.release_and_idle(context, message); } } RecPhase::ConnectingPhotodiode => { - if self.recording.stop_requested && !self.recording.connect_accepted { + if self.recording.pd_rejected { + self.continue_without_photodiode(context); + } else if self.recording.stop_requested && !self.recording.connect_accepted { self.stop_camera(context); } else if self.recording.connect_accepted { if self.recording.stop_requested { @@ -3508,13 +3678,13 @@ impl StageAA1Plugin { } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.pd_rejected = true; - self.note("Timed out connecting the photodiode"); - self.stop_camera(context); + self.note_failure("Timed out connecting the photodiode"); + self.continue_without_photodiode(context); } } RecPhase::AcquiringLease => { if self.recording.pd_rejected { - self.stop_camera(context); + self.continue_without_photodiode(context); } else if self.recording.lease_granted { if self.recording.stop_requested { self.stop_photodiode(context); @@ -3524,8 +3694,8 @@ impl StageAA1Plugin { } else if now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.pd_rejected = true; - self.note("Timed out preparing the photodiode"); - self.stop_camera(context); + self.note_failure("Timed out preparing the photodiode"); + self.continue_without_photodiode(context); } } RecPhase::StartingPhotodiode => { @@ -3545,7 +3715,8 @@ impl StageAA1Plugin { || now_ms.saturating_sub(self.recording.last_activity_ms) > REPLY_TIMEOUT_MS { self.recording.pd_rejected = true; - self.stop_photodiode(context); + self.note_failure("The photodiode did not open its PDQ file"); + self.continue_without_photodiode(context); } } RecPhase::Running => { @@ -3562,7 +3733,7 @@ impl StageAA1Plugin { { self.recording.pd_rejected = true; self.recording.lease_granted = false; - self.note("Timed out saving photodiode data"); + self.note_failure("Timed out saving photodiode data"); self.stop_camera(context); } } @@ -3573,7 +3744,7 @@ impl StageAA1Plugin { { if self.recording.cam_finalized_path.is_none() && !self.recording.cam_rejected { self.recording.cam_rejected = true; - self.note("Timed out saving camera data"); + self.note_failure("Timed out saving camera data"); } self.finish_recording(context); } @@ -3921,6 +4092,39 @@ fn waveform_label(waveform: &WaveformV1) -> String { } } +/// Moves `source` into `dir`, returning the new path when it now lives there. +/// +/// A rename covers the common case (one volume) at zero cost; a cross-volume +/// move falls back to copy-then-delete, and the copy is size-checked before the +/// original goes away so a failed move never loses measurement data. `None` +/// means the file stayed where it was — callers keep the original path. +fn move_into(dir: &Path, source: &str) -> Option { + let source = Path::new(source); + let name = source.file_name()?; + if source.parent() == Some(dir) { + return None; + } + if !source.is_file() { + return None; + } + let destination = dir.join(name); + if destination.exists() { + return None; + } + if std::fs::rename(source, &destination).is_ok() { + return Some(destination.display().to_string()); + } + let copied = std::fs::copy(source, &destination).ok()?; + let expected = source.metadata().ok()?.len(); + if copied != expected { + let _ = std::fs::remove_file(&destination); + return None; + } + // Keeping the original after a verified copy is harmless; losing it is not. + let _ = std::fs::remove_file(source); + Some(destination.display().to_string()) +} + fn sibling_toml(raw_path: &str) -> Option { let path = Path::new(raw_path); let stem = path.file_stem()?.to_string_lossy(); @@ -4080,7 +4284,22 @@ impl Plugin for StageAA1Plugin { PluginDiscontinuity::SettingsChanged => {} PluginDiscontinuity::Seek | PluginDiscontinuity::SourceChanged - | PluginDiscontinuity::HistoryEvicted => self.reset(), + | PluginDiscontinuity::HistoryEvicted => { + // Starting and stopping the host recorder restarts the capture + // pipeline, and the host reports that as SourceChanged. Those + // boundaries are self-inflicted — twice per recording — so they + // must not wipe the row's pilot windows, background floor, or + // the response points collected across a sweep. The event fold + // still resets: that timeline really did restart. + if self.recording.is_active() || self.sweep.is_some() { + self.camera_events.clear(); + self.event_scratch.clear(); + self.camera_markers_us.clear(); + self.bump(); + } else { + self.reset(); + } + } } } @@ -4220,11 +4439,12 @@ impl Plugin for StageAA1Plugin { description: Some( "Records the camera RAW stream and the photodiode PDQ stream together for \ a fixed duration and writes an A1 config sidecar (.toml) linking them. \ - Files are grouped under the measurement id and share an _ \ - stem. Arm the optical drive in the modulation plugin first; A1 only reads \ - its settings — it never drives the Teensy. For everything to land in one \ - place, point the host output folder and the photodiode data folder at the \ - same experiment directory as this folder." + Everything lands under // and shares an \ + _ stem: the RAW and PDQ are gathered here once both are \ + finalized, wherever their own recorders wrote them. Arm the optical \ + drive in the modulation plugin first; A1 only reads its settings — it \ + never drives the Teensy. The photodiode must be connected and have a \ + data directory set, otherwise the recording is refused before it starts." .into(), ), default_open: true, @@ -4233,8 +4453,9 @@ impl Plugin for StageAA1Plugin { key: "output_folder".into(), label: "Output folder".into(), tooltip: Some( - "Directory where the A1 config sidecar is written. Also the \ - recommended shared experiment root for the RAW/PDQ files." + "Experiment directory for this measurement. The config sidecar is \ + written here, and the camera RAW and photodiode PDQ are moved \ + here once finalized, so one measurement is one folder." .into(), ), kind: SettingKind::Path { @@ -5266,8 +5487,9 @@ export_plugin!(StageAA1Plugin); #[cfg(test)] mod tests { use stage_a_plugin_contract::{ - OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, RequestOutcomeV1, - ResponseCommonV1, Sha256V1, StreamIntegrityV1, CONTRACT_VERSION_V1, + FreshnessV1, OwnerInstanceId, PdqFinalizedReceiptV1, PdqStartedReceiptV1, + PhotodiodeStreamV1, RequestOutcomeV1, ResponseCommonV1, Sha256V1, StreamIntegrityV1, + SynchronizationV1, CONTRACT_VERSION_V1, }; use super::*; @@ -5387,6 +5609,7 @@ mod tests { }, active_recording: None, last_finalized_recording: None, + data_dir: Some(std::env::temp_dir().display().to_string()), optical_summary: Some(stage_a_plugin_contract::PhotodiodeOpticalSummaryV1 { run_id: RunId::new("pd-run"), calibration: stage_a_plugin_contract::PhotodiodeCalibrationV1 { @@ -5533,6 +5756,45 @@ mod tests { } } + /// A photodiode summary that passes the pre-flight: connected, unleased, + /// and with somewhere to put the PDQ. + fn ready_photodiode() -> PhotodiodeSummaryV1 { + PhotodiodeSummaryV1 { + contract_version: CONTRACT_VERSION_V1, + owner_instance: OwnerInstanceId::new("pd-test"), + service_revision: 1, + connection: ConnectionStateV1::Connected { + port_label: "mock".into(), + firmware_version: None, + }, + lease: None, + active_run_id: None, + requested_revision: None, + acknowledged_revision: None, + stream: PhotodiodeStreamV1 { + stream_epoch: 1, + sample_range: None, + sample_rate_hz: Some(20_000), + latest_adc_code: Some(1_000), + integrity: StreamIntegrityV1::default(), + level: None, + }, + data_dir: Some("/pd".into()), + active_recording: None, + last_finalized_recording: None, + optical_summary: None, + synchronization: SynchronizationV1::Unsynced { + reason: stage_a_plugin_contract::UnsyncedReasonV1::NoLease, + detail: None, + }, + last_response: None, + freshness: FreshnessV1 { + observed_at_unix_ms: now_unix_ms(), + valid_for_ms: 60_000, + }, + } + } + fn on(timestamp_us: u64) -> CameraEvent { CameraEvent { timestamp_us, @@ -5790,6 +6052,7 @@ mod tests { measurement_id: "A1-row".into(), duration_s: 1, pending_role: Some(RecRole::Normal), + photodiode: Some(ready_photodiode()), ..StageAA1Plugin::default() }; let mut sink = ControlSink::default(); @@ -6919,4 +7182,248 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + fn pd_rejection(request_id: u64, code: &str, message: &str) -> PluginServiceReply { + PluginServiceReply { + request_id, + source_plugin_id: A1_PLUGIN_ID.into(), + target_plugin_id: PHOTODIODE_PLUGIN_ID.into(), + service: SERVICE_STAGE_A_PHOTODIODE_CONTROL_V1.into(), + outcome: PluginServiceOutcome::Rejected { + code: code.into(), + message: message.into(), + }, + } + } + + /// A photodiode that cannot record is caught before the host is recording, + /// so a misconfigured bench no longer leaves a stub RAW behind. + #[test] + fn a_photodiode_without_a_data_directory_is_refused_before_the_camera_starts() { + let mut photodiode = ready_photodiode(); + photodiode.data_dir = None; + let mut plugin = StageAA1Plugin { + output_folder: "/tmp/a1-preflight".into(), + measurement_id: "A1-row".into(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(photodiode), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + + assert_eq!(plugin.recording.phase, RecPhase::Idle); + assert!( + sink.hosts.is_empty(), + "the camera must not start when the PDQ has nowhere to go" + ); + assert!( + plugin.message.contains("Data directory"), + "message={}", + plugin.message + ); + } + + /// The regression this whole coordinator exists for: a photodiode failure + /// used to stop the host recorder immediately, leaving a RAW that was a + /// fraction of the requested duration but reported itself as finalized. + #[test] + fn a_photodiode_failure_keeps_the_camera_recording_for_the_full_duration() { + let folder = std::env::temp_dir().join(format!("a1-camera-only-{}", now_unix_ms())); + let host_dir = folder.join("host-output"); + std::fs::create_dir_all(&host_dir).expect("host dir"); + let mut plugin = StageAA1Plugin { + output_folder: folder.display().to_string(), + measurement_id: "A1-row".into(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(ready_photodiode()), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + let cam_start_req = sink.hosts[0].request_id; + let raw_path = host_dir.join(format!("{}.raw", plugin.recording.stem)); + std::fs::write(&raw_path, b"raw-events").expect("raw file"); + std::fs::write(raw_path.with_extension("toml"), b"biases = true").expect("bias sidecar"); + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_start_req, + outcome: HostCommandOutcome::RecordingStarted { + actual_raw_path: raw_path.display().to_string(), + started_at: "2026-07-25T00:00:00Z".into(), + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let connect = sink.services.last().expect("connect request").clone(); + + // The photodiode refuses to open the stream. + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_rejection( + connect.request_id, + "transport", + "photodiode connection failed", + )], + ..PluginControlInbox::default() + }, + &mut sink, + ); + + assert_eq!( + plugin.recording.phase, + RecPhase::Running, + "the camera must keep recording without the photodiode" + ); + assert_eq!( + sink.hosts.len(), + 1, + "no StopRecording may be sent before the duration elapses" + ); + + // Nothing happens until the fixed duration is actually over. + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!(sink.hosts.len(), 1, "the run is still inside its window"); + + plugin.recording.start_unix_ms = now_unix_ms().saturating_sub(10_000); + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!(plugin.recording.phase, RecPhase::StoppingCamera); + let cam_stop_req = sink.hosts[1].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_stop_req, + outcome: HostCommandOutcome::RecordingFinalized { + actual_raw_path: raw_path.display().to_string(), + size: 10, + sha256: "cd".repeat(32), + duration_us: 10_000_000, + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + + assert_eq!(plugin.recording.phase, RecPhase::Idle); + assert!(!plugin.recording_completed_ok, "the PDQ is missing"); + // The closing message names the cause instead of only "incomplete". + assert!( + plugin.message.contains("photodiode connection failed"), + "message={}", + plugin.message + ); + // Camera RAW, its bias sidecar, and the config all land together. + let measurement_dir = folder.join("A1-row"); + let mut names: Vec = std::fs::read_dir(&measurement_dir) + .expect("measurement folder") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + assert_eq!(names.len(), 3, "names={names:?}"); + assert!(names.iter().any(|name| name.ends_with(".raw"))); + assert!(names.iter().any(|name| name.ends_with("_config.toml"))); + assert!( + !raw_path.exists(), + "the RAW must be moved out of the host output folder" + ); + + let _ = std::fs::remove_dir_all(&folder); + } + + /// PDQ receipts name the path *relative to the photodiode's data directory*, + /// so gathering has to resolve it against the owner's published root before + /// the file can be found and moved. + #[test] + fn a_relative_pdq_label_is_resolved_against_the_photodiode_data_directory() { + let root = std::env::temp_dir().join(format!("a1-gather-{}", now_unix_ms())); + let pd_root = root.join("pd-data"); + std::fs::create_dir_all(pd_root.join("A1-row")).expect("pd dirs"); + std::fs::write(pd_root.join("A1-row/run_pd.pdq"), b"pdq").expect("pdq"); + std::fs::write(pd_root.join("A1-row/run_pd.json"), b"{}").expect("pd sidecar"); + + let mut photodiode = ready_photodiode(); + photodiode.data_dir = Some(pd_root.display().to_string()); + let mut plugin = StageAA1Plugin { + output_folder: root.display().to_string(), + photodiode: Some(photodiode), + ..StageAA1Plugin::default() + }; + plugin.recording.id = "A1-row".into(); + plugin.recording.stem = "run".into(); + plugin.recording.folder = root.display().to_string(); + // Exactly what the owner reports: a label, not a path. + plugin.recording.pd_pdq_path = Some("A1-row/run_pd.pdq".into()); + plugin.recording.pd_sidecar_path = Some("A1-row/run_pd.json".into()); + + plugin.gather_into_measurement_folder(); + + let measurement_dir = root.join("A1-row"); + assert!(measurement_dir.join("run_pd.pdq").is_file()); + assert!(measurement_dir.join("run_pd.json").is_file()); + assert!(!pd_root.join("A1-row/run_pd.pdq").exists()); + // The sidecar records where the file actually ended up. + assert_eq!( + plugin.recording.pd_pdq_path.as_deref(), + Some(measurement_dir.join("run_pd.pdq").display().to_string()).as_deref() + ); + + let _ = std::fs::remove_dir_all(&root); + } + + /// The host restarts the pipeline when A1 starts its own recording and + /// reports it as SourceChanged. That must not wipe the row's science state. + #[test] + fn a_self_inflicted_source_change_keeps_the_rows_science_state() { + let mut plugin = StageAA1Plugin { + response_points: vec![ResponsePoint { + measured_a: 1.0, + q_on: 0.5, + q_off: 0.4, + cycles: 20, + valid_pixels: 10, + }], + pilot_windows: Some(( + PhaseWindow { + start: 0.1, + end: 0.4, + }, + PhaseWindow { + start: 0.6, + end: 0.9, + }, + )), + camera_markers_us: vec![0, 1_000], + ..StageAA1Plugin::default() + }; + plugin.recording.phase = RecPhase::Running; + + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + + assert_eq!(plugin.response_points.len(), 1, "sweep points were wiped"); + assert!(plugin.pilot_windows.is_some(), "pilot windows were wiped"); + assert!( + plugin.camera_markers_us.is_empty(), + "the event timeline really did restart and must reset" + ); + + // Outside a recording the boundary still resets everything. + plugin.recording = Recording::idle(); + plugin.on_discontinuity(PluginDiscontinuity::SourceChanged); + assert!(plugin.response_points.is_empty()); + assert!(plugin.pilot_windows.is_none()); + } } diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 95cbc59..3e39e79 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -1766,6 +1766,10 @@ impl StageAPhotodiodePlugin { requested_revision: self.requested_revision, acknowledged_revision: self.acknowledged_revision, stream, + // Automation clients need this to refuse a coordinated run before + // it starts the camera, instead of failing at BeginRecording. + data_dir: Some(self.data_dir.trim().to_owned()) + .filter(|folder| !folder.is_empty()), active_recording, last_finalized_recording: self.last_finalized_recording.clone(), optical_summary, diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 29b2b33..5028931 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -613,6 +613,11 @@ pub struct PhotodiodeSummaryV1 { pub requested_revision: Option, pub acknowledged_revision: Option, pub stream: PhotodiodeStreamV1, + /// Directory the owner resolves relative PDQ/sidecar paths against. `None` + /// when it is unset, in which case every recording command is rejected — + /// automation clients check this before they start a coordinated run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_dir: Option, pub active_recording: Option, pub last_finalized_recording: Option, pub optical_summary: Option, From 7b78432f3ee78277d2512f505db1e82aa6ffaaf5 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 28 Jul 2026 09:11:42 +0200 Subject: [PATCH 29/30] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20write=20th?= =?UTF-8?q?e=20A1=20PDQ=20straight=20into=20the=20measurement=20folder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gather-after-finalization from the previous commit left the photodiode's own Data directory in the critical path: an A1 run still failed when it was unset, and changing it mid-experiment could move files out from under a measurement. `PdqStartSpecV1` gains an additive `root_dir: Option` — an absolute directory the client wants the recording written below, replacing the owner's configured data directory for that run. Every safety rule the owner already had survives below the new root (the path stays relative, `..` and non-normal components refused, parent components must be real directories rather than symlinks, the resolved target must stay below the root), plus the root itself must be absolute. An A1-driven run therefore no longer depends on the photodiode's Data directory at all, and the pre-flight stops checking it. The camera RAW still has to be gathered after finalization: the host resolves plugin recording paths below its own output directory and rejects absolute ones, and that rule lives in the other repository. The gather now also runs over the PDQ, where it is normally a no-op because the file was opened in place — which means a run that dies before finalization still leaves its PDQ in the measurement folder. PDQ receipts report the label the client requested, so A1 resolves it against the root it named, falling back to the owner's published `data_dir` and preferring whichever exists — an owner too old to honour `root_dir` still yields a correct path. Cherry-picked from fix/stage-a-a1-recording; revises ADR 015 decision 4. --- .../015-stage-a-a1-recording-robustness.md | 68 +++++--- docs/features/stage-a-a1.md | 31 ++-- plugins/stage-a-a1/src/runtime.rs | 123 ++++++++++++--- plugins/stage-a-photodiode/src/lib.rs | 147 ++++++++++++++++-- stage-a-plugin-contract/src/lib.rs | 8 + 5 files changed, 314 insertions(+), 63 deletions(-) diff --git a/docs/adr/015-stage-a-a1-recording-robustness.md b/docs/adr/015-stage-a-a1-recording-robustness.md index 15af6f3..5981e15 100644 --- a/docs/adr/015-stage-a-a1-recording-robustness.md +++ b/docs/adr/015-stage-a-a1-recording-robustness.md @@ -69,24 +69,39 @@ Three separate defects produced that outcome. first, most specific cause; later fallout cannot overwrite it. The closing message reads `Recording incomplete: — metadata saved to `. -4. **A1's output folder becomes authoritative for the whole measurement.** After - both recorders report finalization, A1 moves the RAW, the host's bias sidecar, - and the PDQ and its sidecar into `//`, then writes the - config sidecar with the final paths. This reverses ADR 009's "co-location is a - configuration convention" without touching the host or photodiode path rules: - both files are closed and hashed by the time their receipts arrive, so moving - them afterwards is safe and stays inside each owner's contract. - - The move is a `rename` on one volume and a size-verified copy-then-delete - across volumes. It never overwrites an existing destination and never removes - a source it has not verified; if a move fails, the file stays put and the - sidecar records where it actually is. - - PDQ receipts report the path **label** A1 requested — relative to the - photodiode's data directory — not an absolute path, so A1 resolves it against - the `data_dir` from decision 2 before locating the file. The sidecar records - the resolved absolute path either way, which also fixes the previous ambiguity - of storing a bare relative label under `[files]`. +4. **A1's output folder is the destination for the whole measurement**, reversing + ADR 009's "co-location is a configuration convention". A recording started in + A1 puts every file under `//`, by two mechanisms — chosen + per recorder by how much control that owner grants a client: + + **The PDQ is written there directly.** `PdqStartSpecV1` gains an additive + `root_dir: Option`: an absolute directory the client wants the + recording written below, replacing the owner's configured data directory for + that run. The owner keeps every safety rule it already had below the new root + — the path stays relative, `..` and non-normal components are refused, parent + components must be real directories rather than symlinks, and the resolved + target must stay below the root — and additionally requires the root itself to + be absolute. Consequently an A1-driven run **does not depend on the + photodiode's own Data directory at all**, which is what removed the failure + mode in context item 1; the pre-flight in decision 2 no longer checks it. + + **The camera RAW is moved there after finalization.** The host resolves plugin + recording paths below *its* output directory and rejects absolute paths, and + it lives in the other repository, so A1 cannot name the destination up front. + Instead, once the host reports finalization — at which point the file is closed + and hashed — A1 moves the RAW and the host's bias sidecar into the measurement + folder. A `rename` on one volume, a size-verified copy-then-delete across + volumes; it never overwrites an existing destination and never removes a source + it has not verified. If a move fails the file stays put and the sidecar records + where it actually is. The same gather runs over the PDQ, which is normally a + no-op because it is already in place. + + PDQ receipts report the path **label** the client requested, not an absolute + path, so A1 resolves it against the root it named — falling back to the owner's + published `data_dir` (decision 2) and preferring whichever exists, so an owner + too old to honour `root_dir` still yields a correct path. The sidecar records + the resolved absolute path, which also fixes the previous ambiguity of storing + a bare relative label under `[files]`. 5. **Self-inflicted pipeline restarts no longer wipe the row.** Starting and stopping the host recorder restarts the capture pipeline, which the host @@ -107,6 +122,21 @@ Three separate defects produced that outcome. the host and photodiode settings happen to point, so operators do not have to keep three roots aligned by hand. Aligning them is still harmless — a file already in the destination is left alone. +- The photodiode's Data directory now governs only its *own* manual saves (cache + snapshots, operator-started recordings). A workflow-driven run overrides it, so + changing it mid-experiment cannot move A1's files out from under a measurement. +- A crash mid-run leaves the PDQ in the measurement folder, because it was opened + there. Only the camera RAW depends on surviving to finalization to be gathered; + if a run dies before that, the RAW is left in the host's output directory and + the sidecar (if written) names it there. - Moving a large RAW across volumes copies it. On one volume (the normal case) the move is a metadata operation regardless of file size. -- The contract addition is additive and backward compatible; no ABI change. +- Both contract additions (`data_dir`, `root_dir`) are additive `#[serde(default)]` + fields, backward compatible in both directions; no ABI change and the contract + version stays at 1. Letting a client name an absolute root is a deliberate + widening of what a workflow may ask the owner to do — bounded by keeping every + traversal and symlink check, and by the owner still refusing anything it cannot + resolve below that root. +- Making the camera RAW land directly in the measurement folder would need the + host to accept a plugin-declared recording root. That belongs to `augur-rs` and + is deliberately left out of scope here; the gather makes it unnecessary. diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 8b0f152..6266165 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -81,16 +81,20 @@ the drive to still be where a previous action left it (ADR 013). - `/__pd.pdq` + `_pd.json` — photodiode PDQ + sidecar. - `/__config.toml` — the A1 sidecar. -**Everything lands under `//`.** The two recorders each -write below their own root while recording — the host resolves plugin recording -paths below *its* output directory and rejects absolute ones, the photodiode -resolves PDQ paths below *its* data directory — so once both files are finalized -(closed and hashed) A1 moves them into the measurement folder and records the -final paths in the sidecar (ADR 015). The A1 output folder is therefore the only -setting that decides where a measurement ends up; the host and photodiode roots -no longer have to be kept aligned by hand. A move is a rename on one volume and a -size-verified copy across volumes; a file that cannot be moved stays where it is -and the sidecar points at it there. +**Everything lands under `//`** (ADR 015). That folder is +the only setting deciding where a measurement ends up — the host output root and +the photodiode Data directory no longer have to be kept aligned by hand: + +- **The PDQ and its sidecar are written there directly.** A1 names the + destination root in the start spec (`PdqStartSpecV1::root_dir`), which replaces + the photodiode's own Data directory for that run. An A1-driven recording + therefore does not depend on the photodiode's folder setting at all. +- **The camera RAW and the host's bias `.toml` are moved there after + finalization.** The host resolves plugin recording paths below *its* output + directory and rejects absolute ones, so A1 cannot name the destination up + front; instead it gathers the file once the host reports it closed and hashed. + A rename on one volume, a size-verified copy across volumes. A file that cannot + be moved stays where it is and the sidecar points at it there. **A1 config sidecar** captures: `measurement_id`, file stem, role, start/finalize timestamps, duration; the sweep `[min_a, max_a]`; modulation settings from the @@ -119,9 +123,10 @@ manifest. Every role uses this same lifecycle. **When something is wrong** (ADR 015): - **Before the camera starts**, A1 refuses the recording — writing nothing — if - the photodiode is not reporting status, not connected, has no data directory, - or is leased by someone else. The same hint fills the status `message` cell - while idle, so it is visible before the button is pressed. + the photodiode is not reporting status, is not connected, or is leased by + someone else. The same hint fills the status `message` cell while idle, so it + is visible before the button is pressed. (The photodiode's *Data directory* is + deliberately not among these: A1 supplies the destination itself.) - **If the photodiode fails once the camera is running**, the camera keeps recording for the full requested duration and closes normally. The run is marked camera-only: `recording_completed_ok` stays false (so a sweep stops), diff --git a/plugins/stage-a-a1/src/runtime.rs b/plugins/stage-a-a1/src/runtime.rs index 5133f6f..845f6c6 100644 --- a/plugins/stage-a-a1/src/runtime.rs +++ b/plugins/stage-a-a1/src/runtime.rs @@ -1480,16 +1480,10 @@ impl StageAA1Plugin { connection_label(&photodiode.connection) )); } - if photodiode - .data_dir - .as_ref() - .is_none_or(|folder| folder.trim().is_empty()) - { - return Some( - "Set the photodiode Data directory before recording — the PDQ has nowhere to go" - .into(), - ); - } + // The photodiode's own Data directory is deliberately *not* checked: A1 + // names the destination root in the start spec, so a recording started + // here does not depend on the owner's folder setting at all. + // // A lease held by anyone else means the PDQ is already committed. if let Some(lease) = photodiode.lease.as_ref() { if lease.holder.as_str() != A1_PLUGIN_ID { @@ -1664,6 +1658,10 @@ impl StageAA1Plugin { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: self.recording_metadata(), + // Write the PDQ straight into this measurement's folder rather than + // the photodiode's own data directory: for a recording started here, + // this plugin's output folder is the one that decides where files go. + root_dir: Some(self.recording.folder.clone()), }; let pd_request = self.photodiode_request(PhotodiodeCommandV1::BeginRecording { specification: spec, @@ -1820,20 +1818,30 @@ impl StageAA1Plugin { } } - /// Absolute location of a photodiode-reported recording path. The owner - /// reports paths relative to its own data directory, which it publishes in - /// its summary; an already-absolute path is taken as given. + /// Absolute location of a photodiode-reported recording path. Receipts name + /// the *label* A1 asked for, which is relative to whichever root the owner + /// used: the folder A1 named in the start spec, or — for an owner too old to + /// honour it — the owner's own data directory. Resolve against both and + /// prefer the one that exists. fn resolved_photodiode_path(&self, reported: Option<&str>) -> Option { let reported = reported?; let path = Path::new(reported); if path.is_absolute() { return Some(reported.to_owned()); } - let root = self + let requested = Path::new(&self.recording.folder).join(path); + if requested.exists() { + return Some(requested.display().to_string()); + } + let owner_root = self .photodiode .as_ref() - .and_then(|photodiode| photodiode.data_dir.as_deref())?; - Some(Path::new(root).join(path).display().to_string()) + .and_then(|photodiode| photodiode.data_dir.as_deref()) + .map(|root| Path::new(root).join(path)); + match owner_root { + Some(owner_root) if owner_root.exists() => Some(owner_root.display().to_string()), + _ => Some(requested.display().to_string()), + } } /// Release the photodiode lease (only if we actually hold it) and return to idle. @@ -7199,9 +7207,9 @@ mod tests { /// A photodiode that cannot record is caught before the host is recording, /// so a misconfigured bench no longer leaves a stub RAW behind. #[test] - fn a_photodiode_without_a_data_directory_is_refused_before_the_camera_starts() { + fn a_disconnected_photodiode_is_refused_before_the_camera_starts() { let mut photodiode = ready_photodiode(); - photodiode.data_dir = None; + photodiode.connection = ConnectionStateV1::Disconnected; let mut plugin = StageAA1Plugin { output_folder: "/tmp/a1-preflight".into(), measurement_id: "A1-row".into(), @@ -7217,15 +7225,90 @@ mod tests { assert_eq!(plugin.recording.phase, RecPhase::Idle); assert!( sink.hosts.is_empty(), - "the camera must not start when the PDQ has nowhere to go" + "the camera must not start when the PDQ cannot follow" ); assert!( - plugin.message.contains("Data directory"), + plugin.message.contains("connect it"), "message={}", plugin.message ); } + /// The photodiode's own Data directory is irrelevant to a recording started + /// from A1: A1 names the destination root, so the run proceeds and the PDQ + /// is written into A1's measurement folder. + #[test] + fn the_pdq_start_spec_points_at_the_a1_output_folder() { + let mut photodiode = ready_photodiode(); + photodiode.data_dir = None; + let mut plugin = StageAA1Plugin { + output_folder: "/tmp/a1-destination".into(), + measurement_id: "A1-row".into(), + duration_s: 10, + pending_role: Some(RecRole::Normal), + photodiode: Some(photodiode), + ..StageAA1Plugin::default() + }; + let mut sink = ControlSink::default(); + + control_tick(&mut plugin, PluginControlInbox::default(), &mut sink); + assert_eq!( + plugin.recording.phase, + RecPhase::StartingCamera, + "an unset owner data directory must not block an A1-driven run" + ); + let cam_start_req = sink.hosts[0].request_id; + + control_tick( + &mut plugin, + PluginControlInbox { + host_replies: vec![HostCommandReply { + request_id: cam_start_req, + outcome: HostCommandOutcome::RecordingStarted { + actual_raw_path: "/camera/A1-row/run.raw".into(), + started_at: "2026-07-25T00:00:00Z".into(), + }, + }], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let connect = sink.services.last().expect("connect").clone(); + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(connect.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + let acquire = sink.services.last().expect("lease").clone(); + control_tick( + &mut plugin, + PluginControlInbox { + service_replies: vec![pd_reply(acquire.request_id, None)], + ..PluginControlInbox::default() + }, + &mut sink, + ); + + let begin: PhotodiodeRequestV1 = + serde_json::from_value(sink.services.last().expect("begin").payload.clone()) + .expect("begin envelope"); + let PhotodiodeCommandV1::BeginRecording { specification } = begin.command else { + panic!("expected BeginRecording"); + }; + assert_eq!( + specification.root_dir.as_deref(), + Some("/tmp/a1-destination"), + "the PDQ must be written below the A1 output folder" + ); + assert_eq!( + specification.pdq_path, + format!("A1-row/{}_pd.pdq", plugin.recording.stem) + ); + } + /// The regression this whole coordinator exists for: a photodiode failure /// used to stop the host recorder immediately, leaving a RAW that was a /// fraction of the requested duration but reported itself as finalized. diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 3e39e79..f722815 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -976,10 +976,16 @@ impl StageAPhotodiodePlugin { Ok(PathBuf::from(self.data_dir.trim())) } - /// Resolves a workflow-owned relative evidence path beneath the configured - /// data directory. Existing or newly created parent components must be - /// real directories, never symlinks. - fn resolve_control_path(&self, label: &str, extension: &str) -> Result { + /// Resolves a workflow-owned relative evidence path beneath `root_override` + /// when the client named one, else beneath the configured data directory. + /// Existing or newly created parent components must be real directories, + /// never symlinks — that holds for either root. + fn resolve_control_path( + &self, + label: &str, + extension: &str, + root_override: Option<&str>, + ) -> Result { let relative = Path::new(label); if relative.as_os_str().is_empty() || relative.is_absolute() @@ -995,12 +1001,24 @@ impl StageAPhotodiodePlugin { return Err(format!("workflow path must use the .{extension} extension")); } - let root = self.resolved_data_dir()?; + // A client-named root replaces the data directory entirely: a + // coordinated run keeps every file of one measurement together, and + // the owner's own Data section then has no bearing on it. + let root = match root_override.map(str::trim).filter(|root| !root.is_empty()) { + Some(root) => { + let root = PathBuf::from(root); + if !root.is_absolute() { + return Err("workflow recording root must be an absolute path".into()); + } + root + } + None => self.resolved_data_dir()?, + }; std::fs::create_dir_all(&root) .map_err(|err| format!("creating {} failed: {err}", root.display()))?; let root = root .canonicalize() - .map_err(|err| format!("resolving data directory failed: {err}"))?; + .map_err(|err| format!("resolving recording directory failed: {err}"))?; let mut parent = root.clone(); if let Some(relative_parent) = relative.parent() { for component in relative_parent.components() { @@ -1032,7 +1050,7 @@ impl StageAPhotodiodePlugin { .canonicalize() .map_err(|err| format!("resolving {} failed: {err}", parent.display()))?; if !canonical.starts_with(&root) { - return Err("workflow path escapes the configured data directory".into()); + return Err("workflow path escapes the recording directory".into()); } } } @@ -1090,11 +1108,12 @@ impl StageAPhotodiodePlugin { true, )); } + let root = specification.root_dir.as_deref(); let pdq_path = self - .resolve_control_path(&specification.pdq_path, "pdq") + .resolve_control_path(&specification.pdq_path, "pdq", root) .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; let sidecar_path = self - .resolve_control_path(&specification.sidecar_path, "json") + .resolve_control_path(&specification.sidecar_path, "json", root) .map_err(|message| service_error(ServiceErrorCodeV1::InvalidPath, message, false))?; if pdq_path == sidecar_path { return Err(service_error( @@ -1768,8 +1787,7 @@ impl StageAPhotodiodePlugin { stream, // Automation clients need this to refuse a coordinated run before // it starts the camera, instead of failing at BeginRecording. - data_dir: Some(self.data_dir.trim().to_owned()) - .filter(|folder| !folder.is_empty()), + data_dir: Some(self.data_dir.trim().to_owned()).filter(|folder| !folder.is_empty()), active_recording, last_finalized_recording: self.last_finalized_recording.clone(), optical_summary, @@ -4408,6 +4426,109 @@ mod tests { assert!(plugin.set_setting("mode", json!("RAW")).is_err()); } + /// A workflow client that names its own recording root gets the PDQ written + /// there, and the owner's Data directory stops being involved at all — that + /// is what lets one coordinated run keep every file in one folder. + #[test] + fn a_client_named_root_overrides_the_data_directory() { + let dir = temp_dir("client-root"); + let mut plugin = live_plugin(); + plugin.port_hint = "mock".into(); + // Deliberately unset: it must not be consulted. + plugin.data_dir = String::new(); + plugin.connect(); + let acquire = service_request( + &plugin, + 20, + "workflow-a", + PhotodiodeCommandV1::AcquireLease { ttl_ms: 10_000 }, + None, + ); + assert!(matches!( + plugin + .handle_service_request(&acquire, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + )); + + let begin = service_request( + &plugin, + 21, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "A1-row/run_pd.pdq".into(), + sidecar_path: "A1-row/run_pd.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + root_dir: Some(dir.display().to_string()), + }, + }, + Some(1), + ); + assert!( + matches!( + plugin + .handle_service_request(&begin, &live_execution()) + .outcome, + PluginServiceOutcome::Accepted { .. } + ), + "a client-named root must not need the owner's data directory" + ); + assert!(dir.join("A1-row/run_pd.pdq").is_file()); + + // Traversal is still refused below a client-named root. + let escape = service_request( + &plugin, + 22, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "../escape.pdq".into(), + sidecar_path: "A1-row/escape.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + root_dir: Some(dir.display().to_string()), + }, + }, + Some(2), + ); + assert!(matches!( + plugin + .handle_service_request(&escape, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + + // A relative root is refused outright. + let relative_root = service_request( + &plugin, + 23, + "workflow-a", + PhotodiodeCommandV1::BeginRecording { + specification: PdqStartSpecV1 { + pdq_path: "A1-row/other_pd.pdq".into(), + sidecar_path: "A1-row/other_pd.json".into(), + expected_sample_rate_hz: None, + expected_stream_epoch: None, + metadata: BTreeMap::new(), + root_dir: Some("relative/root".into()), + }, + }, + Some(3), + ); + assert!(matches!( + plugin + .handle_service_request(&relative_root, &live_execution()) + .outcome, + PluginServiceOutcome::Rejected { .. } + )); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn named_recording_rejects_unsafe_paths_and_returns_final_receipt() { let dir = temp_dir("named"); @@ -4440,6 +4561,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::new(), + root_dir: None, }, }, Some(1), @@ -4462,6 +4584,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::from([("workflow".into(), "A1".into())]), + root_dir: None, }, }, Some(1), @@ -4511,6 +4634,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::new(), + root_dir: None, }, }, Some(3), @@ -4551,6 +4675,7 @@ mod tests { expected_sample_rate_hz: None, expected_stream_epoch: None, metadata: BTreeMap::new(), + root_dir: None, }, }, Some(1), diff --git a/stage-a-plugin-contract/src/lib.rs b/stage-a-plugin-contract/src/lib.rs index 5028931..97b7aca 100644 --- a/stage-a-plugin-contract/src/lib.rs +++ b/stage-a-plugin-contract/src/lib.rs @@ -440,6 +440,14 @@ pub struct PdqStartSpecV1 { pub expected_sample_rate_hz: Option, pub expected_stream_epoch: Option, pub metadata: BTreeMap, + /// Absolute directory the workflow client wants this recording written + /// below, so a coordinated run can put every file in one measurement + /// folder instead of the owner's own data directory. `None` keeps the + /// owner's configured data directory. `pdq_path`/`sidecar_path` stay + /// relative to whichever root applies, and the owner still refuses + /// traversal and symlinked path components below it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_dir: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] From bcd0e6609bb29fe2cbb578bf460492ed768b71d8 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 28 Jul 2026 09:12:36 +0200 Subject: [PATCH 30/30] =?UTF-8?q?docs(stage-a):=20=F0=9F=93=9D=20index=20t?= =?UTF-8?q?he=20recording-robustness=20ADR=20after=20its=20renumber=20to?= =?UTF-8?q?=20015?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/features/stage-a-a1.md | 3 +++ plugins/stage-a-a1/README.md | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 6266165..9a187b4 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -2,9 +2,12 @@ - **Crate:** `plugins/stage-a-a1` (`augur-plugin-stage-a-a1`) - **Status:** Recording coordinator + live quicklooks + amplitude sweep + `a₀` lock + + unattended frequency ladder - **Design:** [ADR 009](../adr/009-stage-a-a1-recording-coordinator.md), [ADR 010](../adr/010-stage-a-a1-amplitude-sweep.md) (sweep + button press forwarding), + [ADR 015](../adr/015-stage-a-a1-recording-robustness.md) (one folder, full + duration, named failures), [ADR 014](../adr/014-stage-a-a1-frequency-ladder.md) (the unattended ladder), [ADR 013](../adr/013-stage-a-a1-event-count-depth-lock.md) (exact-event-count `a₀` lock) diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 11e3e4d..322ff7d 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -74,6 +74,8 @@ See [docs/features/stage-a-a1.md](../../docs/features/stage-a-a1.md) for the ful [docs/features/stage-a-a1-event-count.md](../../docs/features/stage-a-a1-event-count.md) plus [ADR 013](../../docs/adr/013-stage-a-a1-event-count-depth-lock.md) for the `a₀` lock, [ADR 014](../../docs/adr/014-stage-a-a1-frequency-ladder.md) for the unattended -frequency ladder, and +frequency ladder, +[ADR 015](../../docs/adr/015-stage-a-a1-recording-robustness.md) for the +recording coordinator's one-folder/full-duration guarantees, and [docs/features/stage-a-a1-automation.md](../../docs/features/stage-a-a1-automation.md) for the planned amplitude-sweep automation on top of this.