diff --git a/Cargo.lock b/Cargo.lock index 5a2084bb..19747146 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,7 +809,7 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.14.2" +version = "0.14.3" dependencies = [ "alloy", "ant-protocol", diff --git a/Cargo.toml b/Cargo.toml index ba489e41..01d5af24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-node" -version = "0.14.2" +version = "0.14.3" edition = "2021" authors = ["David Irvine "] description = "Pure quantum-proof network node for the Autonomi decentralized network" diff --git a/src/replication/admission.rs b/src/replication/admission.rs index 3b99a802..944e7b47 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -95,12 +95,16 @@ pub async fn admit_hints( rejected_keys: Vec::new(), }; - // Track all processed keys to deduplicate within and across sets. - let mut seen = HashSet::new(); + // Track replica outcomes separately. A replica hint wins over a duplicate + // paid hint only when it is actually admitted; if local routing churn + // rejects the replica side, the paid-list path still gets a chance. + let mut seen_replica = HashSet::new(); + let mut admitted_replica = HashSet::new(); + let mut rejected_replica = Vec::new(); // Process replica hints. for &key in replica_hints { - if !seen.insert(key) { + if !seen_replica.insert(key) { continue; } @@ -110,6 +114,7 @@ pub async fn admit_hints( if already_local || already_pending { result.replica_keys.push(key); + admitted_replica.insert(key); continue; } @@ -122,15 +127,23 @@ pub async fn admit_hints( .await { result.replica_keys.push(key); + admitted_replica.insert(key); } else { - result.rejected_keys.push(key); + rejected_replica.push(key); } } - // Process paid hints. Cross-set dedup is handled by `seen` — any key - // already processed in the replica-hints loop above is skipped here. + // Process paid hints. A key already admitted as a replica remains a + // replica-pipeline key. If the replica path rejected it, however, paid-list + // admission can still authorize metadata convergence for churned views. + let mut seen_paid = HashSet::new(); + let mut admitted_paid = HashSet::new(); + let mut rejected_paid = Vec::new(); for &key in paid_hints { - if !seen.insert(key) { + if !seen_paid.insert(key) { + continue; + } + if admitted_replica.contains(&key) { continue; } @@ -139,16 +152,25 @@ pub async fn admit_hints( if already_paid { result.paid_only_keys.push(key); + admitted_paid.insert(key); continue; } if is_in_paid_close_group(self_id, &key, p2p_node, config.paid_list_close_group_size).await { result.paid_only_keys.push(key); - } else { + admitted_paid.insert(key); + } else if !seen_replica.contains(&key) { + rejected_paid.push(key); + } + } + + for key in rejected_replica { + if !admitted_paid.contains(&key) { result.rejected_keys.push(key); } } + result.rejected_keys.extend(rejected_paid); result } @@ -225,27 +247,22 @@ mod tests { #[test] fn deduplication_across_sets() { - // If a key appears in replica_hints AND paid_hints, the paid entry - // is skipped because seen already contains it from replica processing. + // If a key appears in replica_hints AND paid_hints and the replica + // side is admitted, the paid entry is skipped because the stronger + // replica pipeline already covers paid-list convergence. let key = xor_name_from_byte(0xFF); let replica_hints = vec![key]; let paid_hints = vec![key]; - let replica_set: HashSet = replica_hints.iter().copied().collect(); - let mut seen: HashSet = HashSet::new(); + let mut admitted_replica: HashSet = HashSet::new(); - // Process replica hints first. for &k in &replica_hints { - seen.insert(k); + admitted_replica.insert(k); } - // Process paid hints: key is already in `seen` AND in `replica_set`. let mut paid_admitted = Vec::new(); for &k in &paid_hints { - if !seen.insert(k) { - continue; // duplicate - } - if replica_set.contains(&k) { + if admitted_replica.contains(&k) { continue; // cross-set precedence } paid_admitted.push(k); @@ -257,6 +274,48 @@ mod tests { ); } + #[test] + fn paid_hint_survives_duplicate_replica_rejection() { + // With churned local views, a sender may believe we are in the storage + // close group while our own view rejects the replica hint. If the same + // key also arrives as a paid hint, paid-list admission must still run. + let key = xor_name_from_byte(0xEE); + let replica_hints = vec![key]; + let paid_hints = vec![key]; + + let mut seen_replica = HashSet::new(); + let admitted_replica: HashSet = HashSet::new(); + let mut rejected_replica = Vec::new(); + + for &k in &replica_hints { + if seen_replica.insert(k) { + rejected_replica.push(k); + } + } + + let mut admitted_paid = HashSet::new(); + for &k in &paid_hints { + if admitted_replica.contains(&k) { + continue; + } + let in_paid_close_group = true; + if in_paid_close_group { + admitted_paid.insert(k); + } + } + + assert!( + admitted_paid.contains(&key), + "paid hint must be considered after duplicate replica rejection" + ); + assert!( + rejected_replica + .into_iter() + .all(|k| admitted_paid.contains(&k)), + "replica rejection should not remain terminal after paid admission" + ); + } + #[test] fn admission_result_empty_inputs() { let result = AdmissionResult { diff --git a/src/replication/audit.rs b/src/replication/audit.rs index 3bfccb65..860afbe1 100644 --- a/src/replication/audit.rs +++ b/src/replication/audit.rs @@ -11,6 +11,8 @@ use rand::seq::SliceRandom; use rand::Rng; use crate::ant_protocol::XorName; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::{self, AuditFailureClass, AuditType}; use crate::replication::config::{ReplicationConfig, REPLICATION_PROTOCOL_ID}; use crate::replication::protocol::{ compute_audit_digest, AuditChallenge, AuditResponse, ReplicationMessage, @@ -42,6 +44,10 @@ pub enum AuditTickResult { Failed { /// Evidence of the failure for trust engine. evidence: FailureEvidence, + /// Node-local no-response class for metrics/logs. This is deliberately + /// kept out of [`FailureEvidence`] so no serialized evidence format or + /// wire protocol changes. + no_response_class: Option<&'static str>, }, /// Audit target claimed bootstrapping. BootstrapClaim { @@ -68,21 +74,15 @@ fn first_challenged_key_label(keys: &[XorName]) -> String { /// The current core networking layer can wrap request deadline timeouts inside /// display strings, so this deliberately remains a bounded heuristic for logs /// rather than protocol/trust semantics. -fn classify_audit_send_error(error: &str) -> &'static str { - let lower = error.to_ascii_lowercase(); - if lower.contains("timed out") || lower.contains("timeout") { - "timeout" - } else if lower.contains("peer not found") || lower.contains("no channel") { - "peer_unavailable" - } else if lower.contains("connection") || lower.contains("connect") || lower.contains("dial") { - "connection_failed" - } else if lower.contains("closed") || lower.contains("dropped") { - "connection_closed" - } else if lower.contains("transport") { - "transport_error" - } else { - "other" - } +fn classify_audit_send_error(error: &str) -> (&'static str, AuditFailureClass) { + audit_metrics::classify_audit_send_error(error) +} + +pub(crate) fn responsible_audit_response_timeout( + config: &ReplicationConfig, + key_count: usize, +) -> std::time::Duration { + config.audit_response_timeout(key_count) } // --------------------------------------------------------------------------- @@ -105,12 +105,14 @@ pub async fn audit_tick( is_bootstrapping: bool, ) -> AuditTickResult { let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); audit_tick_with_repair_proofs( p2p_node, storage, config, sync_history, &repair_proofs, + &audit_challenge_coordinator, 0, is_bootstrapping, ) @@ -123,13 +125,18 @@ pub async fn audit_tick( /// compatibility [`audit_tick`] wrapper passes an empty proof table, so direct /// callers that have not adopted repair proofs remain conservative and do not /// audit peers for unproven keys. -#[allow(clippy::implicit_hasher, clippy::too_many_lines)] +#[allow( + clippy::implicit_hasher, + clippy::too_many_arguments, + clippy::too_many_lines +)] pub async fn audit_tick_with_repair_proofs( p2p_node: &Arc, storage: &Arc, config: &ReplicationConfig, sync_history: &HashMap, repair_proofs: &Arc>, + audit_challenge_coordinator: &Arc, current_sync_epoch: u64, is_bootstrapping: bool, ) -> AuditTickResult { @@ -238,8 +245,12 @@ pub async fn audit_tick_with_repair_proofs( } }; + let Some(_slot) = audit_challenge_coordinator.acquire(challenged_peer).await else { + warn!("Audit: failed to acquire outbound audit coordinator slot for {challenged_peer}"); + return AuditTickResult::Idle; + }; let encoded_len = encoded.len(); - let audit_timeout = config.audit_response_timeout(peer_keys.len()); + let audit_timeout = responsible_audit_response_timeout(config, peer_keys.len()); let audit_started = Instant::now(); let response = match p2p_node .send_request( @@ -252,15 +263,20 @@ pub async fn audit_tick_with_repair_proofs( { Ok(resp) => resp, Err(e) => { + let send_error = e.to_string(); + let (send_error_class, audit_failure_class) = classify_audit_send_error(&send_error); + audit_metrics::record_audit_no_response( + AuditType::ResponsibleChunk, + audit_failure_class, + ); if enabled!(crate::logging::Level::WARN) { let elapsed = audit_started.elapsed(); - let send_error = e.to_string(); - let send_error_class = classify_audit_send_error(&send_error); let first_key = first_challenged_key_label(&peer_keys); warn!( - audit_type = "responsible_chunk", + audit_type = AuditType::ResponsibleChunk.as_str(), audit_phase = "challenge_send", audit_outcome = "send_request_failed", + audit_failure_class = audit_failure_class.as_str(), challenged_peer = %challenged_peer, challenge_id, key_count = peer_keys.len(), @@ -269,7 +285,8 @@ pub async fn audit_tick_with_repair_proofs( first_key = %first_key, encoded_len, send_error_class, - "Audit challenge send_request failed: audit_type=responsible_chunk, audit_phase=challenge_send, audit_outcome=send_request_failed, challenged_peer={challenged_peer}, challenge_id={challenge_id}, key_count={}, timeout_ms={}, elapsed_ms={}, first_key={first_key}, encoded_len={encoded_len}, send_error_class={send_error_class}", + "Audit challenge send_request failed: audit_type=responsible_chunk, audit_phase=challenge_send, audit_outcome=send_request_failed, audit_failure_class={}, challenged_peer={challenged_peer}, challenge_id={challenge_id}, key_count={}, timeout_ms={}, elapsed_ms={}, first_key={first_key}, encoded_len={encoded_len}, send_error_class={send_error_class}", + audit_failure_class.as_str(), peer_keys.len(), audit_timeout.as_millis(), elapsed.as_millis(), @@ -279,13 +296,16 @@ pub async fn audit_tick_with_repair_proofs( challenged_peer = %challenged_peer, challenge_id, send_error = %e, + audit_failure_class = audit_failure_class.as_str(), "Audit challenge raw send_request error" ); - // Timeout — need responsibility confirmation before penalty. + // No-response verdicts still use the existing Timeout evidence + // reason; the class is node-local observability only. return handle_audit_timeout( &challenged_peer, challenge_id, &peer_keys, + audit_failure_class, p2p_node, config, ) @@ -587,6 +607,7 @@ async fn verify_digests( &failed_keys, AuditFailureReason::DigestMismatch, keys.len(), + None, p2p_node, config, ) @@ -617,18 +638,21 @@ async fn handle_audit_failure( &failures, reason, failed_keys.len(), + None, p2p_node, config, ) .await } +#[allow(clippy::too_many_arguments)] async fn handle_classified_audit_failure( challenged_peer: &PeerId, challenge_id: u64, failed_keys: &[AuditKeyFailure], reason: AuditFailureReason, challenged_key_count: usize, + no_response_class: Option<&'static str>, p2p_node: &Arc, config: &ReplicationConfig, ) -> AuditTickResult { @@ -679,7 +703,10 @@ async fn handle_classified_audit_failure( reason, }; - AuditTickResult::Failed { evidence } + AuditTickResult::Failed { + evidence, + no_response_class, + } } /// Handle audit timeout (no response received). @@ -687,14 +714,22 @@ async fn handle_audit_timeout( challenged_peer: &PeerId, challenge_id: u64, keys: &[XorName], + no_response_class: AuditFailureClass, p2p_node: &Arc, config: &ReplicationConfig, ) -> AuditTickResult { - handle_audit_failure( + let failures = keys + .iter() + .copied() + .map(AuditKeyFailure::unclassified) + .collect::>(); + handle_classified_audit_failure( challenged_peer, challenge_id, - keys, + &failures, AuditFailureReason::Timeout, + keys.len(), + Some(no_response_class.as_str()), p2p_node, config, ) @@ -822,25 +857,32 @@ mod tests { fn classify_audit_send_error_uses_bounded_classes() { assert_eq!( classify_audit_send_error("request to peer timed out after 10s"), - "timeout" + ("response_timeout", AuditFailureClass::Timeout) ); assert_eq!( classify_audit_send_error("peer not found in active channels"), - "peer_unavailable" + ("peer_unavailable", AuditFailureClass::Unreachable) ); assert_eq!( classify_audit_send_error("dial failed for all candidate addresses"), - "connection_failed" + ("connection_failed", AuditFailureClass::Unreachable) ); assert_eq!( classify_audit_send_error("response receiver dropped before delivery"), - "connection_closed" + ("connection_closed", AuditFailureClass::Unreachable) ); assert_eq!( classify_audit_send_error("transport stream error"), - "transport_error" + ("transport_error", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("operation timed out after 10s"), + ("transport_timeout", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("unexpected error"), + ("other", AuditFailureClass::Unreachable) ); - assert_eq!(classify_audit_send_error("unexpected error"), "other"); } /// Create a test `LmdbStorage` backed by a temp directory. diff --git a/src/replication/audit_coordinator.rs b/src/replication/audit_coordinator.rs new file mode 100644 index 00000000..c0858f27 --- /dev/null +++ b/src/replication/audit_coordinator.rs @@ -0,0 +1,233 @@ +//! Auditor-side per-target admission for outbound `AuditChallenge`s. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use saorsa_core::identity::PeerId; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +/// Maximum concurrent `AuditChallenge` requests this auditor may have in +/// flight to one target peer. +/// +/// This intentionally matches the per-source digest admission cap enforced by +/// the already-deployed fleet. Waiting here happens before the response +/// deadline starts, so excess local bursts are serialized instead of being +/// converted into guaranteed remote admission drops and false timeouts. +pub(crate) const MAX_CONCURRENT_AUDIT_CHALLENGES_PER_TARGET: usize = 2; + +#[derive(Debug)] +struct TargetLimiter { + semaphore: Arc, + references: usize, +} + +/// Shared limiter for all auditor-side flows that send `AuditChallenge`. +#[derive(Debug, Default)] +pub struct AuditChallengeCoordinator { + targets: Mutex>, +} + +/// Permit held while one outbound challenge is in flight. +#[derive(Debug)] +pub(crate) struct AuditChallengePermit { + coordinator: Arc, + peer: PeerId, + permit: Option, +} + +/// RAII guard that releases a counted target reference unless disarmed. +/// +/// A reference is counted *before* `acquire` awaits the per-target semaphore. +/// Holding this guard across that await guarantees the reference is released +/// even if the future is dropped while parked in the wait queue (task abort, +/// enclosing `timeout`, or a racing `select!` branch). On a successful acquire +/// the guard is disarmed and the returned [`AuditChallengePermit`] assumes the +/// release on its own drop. +struct ReferenceGuard<'a> { + coordinator: &'a AuditChallengeCoordinator, + peer: PeerId, + armed: bool, +} + +impl ReferenceGuard<'_> { + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for ReferenceGuard<'_> { + fn drop(&mut self) { + if self.armed { + self.coordinator.release_reference(self.peer); + } + } +} + +impl AuditChallengeCoordinator { + /// Create an empty coordinator. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Wait for a target-peer slot. Returns `None` only if the internal + /// semaphore was closed, which the coordinator never does in production. + pub(crate) async fn acquire(self: &Arc, peer: PeerId) -> Option { + let semaphore = { + let mut targets = self.lock_targets(); + let entry = targets.entry(peer).or_insert_with(|| TargetLimiter { + semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_AUDIT_CHALLENGES_PER_TARGET)), + references: 0, + }); + entry.references = entry.references.saturating_add(1); + Arc::clone(&entry.semaphore) + }; + + // The reference is now counted. Hold it under an RAII guard across the + // await so a dropped/cancelled future still releases it; a closed + // semaphore releases it via the same guard on the early return. + let mut reference_guard = ReferenceGuard { + coordinator: self, + peer, + armed: true, + }; + + let Ok(permit) = semaphore.acquire_owned().await else { + return None; + }; + + // Hand the release off to the permit's own drop. + reference_guard.disarm(); + Some(AuditChallengePermit { + coordinator: Arc::clone(self), + peer, + permit: Some(permit), + }) + } + + #[cfg(test)] + pub(crate) fn tracked_target_count(&self) -> usize { + self.lock_targets().len() + } + + fn release_reference(&self, peer: PeerId) { + let mut targets = self.lock_targets(); + let Some(entry) = targets.get_mut(&peer) else { + return; + }; + entry.references = entry.references.saturating_sub(1); + if entry.references == 0 { + targets.remove(&peer); + } + } + + fn lock_targets(&self) -> std::sync::MutexGuard<'_, HashMap> { + match self.targets.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } +} + +impl Drop for AuditChallengePermit { + fn drop(&mut self) { + let _permit = self.permit.take(); + self.coordinator.release_reference(self.peer); + } +} + +#[cfg(test)] +#[allow(clippy::panic)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use super::*; + + const PEER_A: [u8; 32] = [0xA1; 32]; + const PEER_B: [u8; 32] = [0xB2; 32]; + const SHORT_WAIT: Duration = Duration::from_millis(50); + + fn peer(bytes: [u8; 32]) -> PeerId { + PeerId::from_bytes(bytes) + } + + #[tokio::test] + async fn excess_challenges_wait_and_are_not_dropped() { + let coordinator = Arc::new(AuditChallengeCoordinator::new()); + let target = peer(PEER_A); + let first = coordinator.acquire(target).await; + let second = coordinator.acquire(target).await; + assert!(first.is_some()); + assert!(second.is_some()); + + let acquired = Arc::new(AtomicUsize::new(0)); + let acquired_clone = Arc::clone(&acquired); + let coordinator_clone = Arc::clone(&coordinator); + let waiting = tokio::spawn(async move { + let permit = coordinator_clone.acquire(target).await; + if permit.is_some() { + acquired_clone.fetch_add(1, Ordering::SeqCst); + } + permit + }); + + tokio::time::sleep(SHORT_WAIT).await; + assert_eq!(acquired.load(Ordering::SeqCst), 0); + + drop(first); + let third = tokio::time::timeout(SHORT_WAIT, waiting).await; + assert!(third.is_ok()); + assert_eq!(acquired.load(Ordering::SeqCst), 1); + + drop(second); + if let Ok(joined) = third { + match joined { + Ok(permit) => drop(permit), + Err(e) => panic!("waiting task failed: {e}"), + } + } + assert_eq!(coordinator.tracked_target_count(), 0); + } + + #[tokio::test] + async fn cross_peer_parallelism_is_preserved() { + let coordinator = Arc::new(AuditChallengeCoordinator::new()); + let target_a = peer(PEER_A); + let target_b = peer(PEER_B); + let first_a = coordinator.acquire(target_a).await; + let second_a = coordinator.acquire(target_a).await; + assert!(first_a.is_some()); + assert!(second_a.is_some()); + + let coordinator_clone = Arc::clone(&coordinator); + let acquired_b = tokio::spawn(async move { coordinator_clone.acquire(target_b).await }); + let result = tokio::time::timeout(SHORT_WAIT, acquired_b).await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn cancelled_wait_releases_reference() { + let coordinator = Arc::new(AuditChallengeCoordinator::new()); + let target = peer(PEER_A); + // Saturate the target so a third acquire parks in the wait queue. + let first = coordinator.acquire(target).await; + let second = coordinator.acquire(target).await; + assert!(first.is_some()); + assert!(second.is_some()); + + // A parked acquire whose future is dropped mid-await must not leak the + // reference it counted before parking. + let parked = coordinator.acquire(target); + let dropped = tokio::time::timeout(SHORT_WAIT, parked).await; + assert!(dropped.is_err(), "third acquire should still be parked"); + + drop(first); + drop(second); + assert_eq!( + coordinator.tracked_target_count(), + 0, + "cancelled wait leaked a target reference" + ); + } +} diff --git a/src/replication/audit_metrics.rs b/src/replication/audit_metrics.rs new file mode 100644 index 00000000..b5bdfa16 --- /dev/null +++ b/src/replication/audit_metrics.rs @@ -0,0 +1,203 @@ +//! Lightweight node-local counters and labels for audit observability. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +/// In-scope audit issuer type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditType { + /// Periodic responsible-chunk audit. + ResponsibleChunk, + /// Prune-confirmation audit. + Prune, + /// ADR-0003 fresh-replication possession check. + Possession, +} + +/// Node-local class for no-response audit verdicts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditFailureClass { + /// The request was delivered but no response arrived before the deadline. + Timeout, + /// The request could not be delivered to the target peer. + Unreachable, +} + +/// Responder-side admission class. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuditResponderClass { + /// Digest-only `AuditChallenge`. + Digest, + /// Subtree proof challenge. + Subtree, + /// Subtree byte-serving challenge. + Byte, +} + +static RESPONSIBLE_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +static RESPONSIBLE_UNREACHABLE: AtomicU64 = AtomicU64::new(0); +static PRUNE_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +static PRUNE_UNREACHABLE: AtomicU64 = AtomicU64::new(0); +static POSSESSION_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +static POSSESSION_UNREACHABLE: AtomicU64 = AtomicU64::new(0); + +static REPLICATION_EVENT_LAGGED: AtomicU64 = AtomicU64::new(0); +static DIGEST_ADMISSION_DROPS: AtomicU64 = AtomicU64::new(0); +static SUBTREE_ADMISSION_DROPS: AtomicU64 = AtomicU64::new(0); +static BYTE_ADMISSION_DROPS: AtomicU64 = AtomicU64::new(0); + +static DIGEST_DISPATCH_LATENCY_COUNT: AtomicU64 = AtomicU64::new(0); +static DIGEST_DISPATCH_LATENCY_TOTAL_MS: AtomicU64 = AtomicU64::new(0); +static DIGEST_DISPATCH_LATENCY_MAX_MS: AtomicU64 = AtomicU64::new(0); + +#[cfg(feature = "logging")] +impl AuditType { + /// Stable structured-log label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ResponsibleChunk => "responsible_chunk", + Self::Prune => "prune", + Self::Possession => "possession", + } + } +} + +impl AuditFailureClass { + /// Stable structured-log label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::Unreachable => "unreachable", + } + } +} + +#[cfg(feature = "logging")] +impl AuditResponderClass { + /// Stable structured-log label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Digest => "digest", + Self::Subtree => "subtree", + Self::Byte => "byte", + } + } +} + +/// Best-effort coarse class for the transport/request error returned by +/// `P2PNode::send_request`. +/// +/// The current core networking layer exposes request-response delivery failure +/// and response-deadline expiry through display strings. Keep this bounded and +/// local to observability: trust evidence still uses the existing +/// `AuditFailureReason::Timeout` wire-compatible reason. +#[must_use] +pub fn classify_audit_send_error(error: &str) -> (&'static str, AuditFailureClass) { + let lower = error.to_ascii_lowercase(); + if lower.contains("request to") && lower.contains("timed out") { + ("response_timeout", AuditFailureClass::Timeout) + } else if lower.contains("peer not found") || lower.contains("no channel") { + ("peer_unavailable", AuditFailureClass::Unreachable) + } else if lower.contains("connection") || lower.contains("connect") || lower.contains("dial") { + ("connection_failed", AuditFailureClass::Unreachable) + } else if lower.contains("closed") || lower.contains("dropped") { + ("connection_closed", AuditFailureClass::Unreachable) + } else if lower.contains("transport") { + ("transport_error", AuditFailureClass::Unreachable) + } else if lower.contains("timed out") || lower.contains("timeout") { + ("transport_timeout", AuditFailureClass::Unreachable) + } else { + ("other", AuditFailureClass::Unreachable) + } +} + +pub fn record_audit_no_response(audit_type: AuditType, class: AuditFailureClass) { + match (audit_type, class) { + (AuditType::ResponsibleChunk, AuditFailureClass::Timeout) => { + RESPONSIBLE_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + (AuditType::ResponsibleChunk, AuditFailureClass::Unreachable) => { + RESPONSIBLE_UNREACHABLE.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Prune, AuditFailureClass::Timeout) => { + PRUNE_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Prune, AuditFailureClass::Unreachable) => { + PRUNE_UNREACHABLE.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Possession, AuditFailureClass::Timeout) => { + POSSESSION_TIMEOUTS.fetch_add(1, Ordering::Relaxed); + } + (AuditType::Possession, AuditFailureClass::Unreachable) => { + POSSESSION_UNREACHABLE.fetch_add(1, Ordering::Relaxed); + } + } +} + +pub fn record_replication_event_lagged(missed: u64) { + REPLICATION_EVENT_LAGGED.fetch_add(missed, Ordering::Relaxed); +} + +pub fn record_admission_drop(class: AuditResponderClass) { + match class { + AuditResponderClass::Digest => { + DIGEST_ADMISSION_DROPS.fetch_add(1, Ordering::Relaxed); + } + AuditResponderClass::Subtree => { + SUBTREE_ADMISSION_DROPS.fetch_add(1, Ordering::Relaxed); + } + AuditResponderClass::Byte => { + BYTE_ADMISSION_DROPS.fetch_add(1, Ordering::Relaxed); + } + } +} + +pub fn record_digest_dispatch_latency(latency: Duration) { + let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX); + DIGEST_DISPATCH_LATENCY_COUNT.fetch_add(1, Ordering::Relaxed); + DIGEST_DISPATCH_LATENCY_TOTAL_MS.fetch_add(latency_ms, Ordering::Relaxed); + update_max(&DIGEST_DISPATCH_LATENCY_MAX_MS, latency_ms); +} + +#[cfg(test)] +pub fn replication_event_lagged_total() -> u64 { + REPLICATION_EVENT_LAGGED.load(Ordering::Relaxed) +} + +fn update_max(max: &AtomicU64, value: u64) { + let mut current = max.load(Ordering::Relaxed); + while value > current { + match max.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => break, + Err(observed) => current = observed, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn send_error_classification_splits_timeout_from_unreachable() { + assert_eq!( + classify_audit_send_error("Request to peer on /replication timed out after 4s"), + ("response_timeout", AuditFailureClass::Timeout) + ); + assert_eq!( + classify_audit_send_error("peer not found in active channels"), + ("peer_unavailable", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("dial failed for all candidate addresses"), + ("connection_failed", AuditFailureClass::Unreachable) + ); + assert_eq!( + classify_audit_send_error("operation timed out after 10s"), + ("transport_timeout", AuditFailureClass::Unreachable) + ); + } +} diff --git a/src/replication/bootstrap.rs b/src/replication/bootstrap.rs index 65025142..06b6bad3 100644 --- a/src/replication/bootstrap.rs +++ b/src/replication/bootstrap.rs @@ -298,12 +298,14 @@ mod tests { let mut queues = ReplicationQueues::new(); // Put the bootstrap key into the pending-verify queue. + let now = Instant::now(); let entry = VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: saorsa_core::identity::PeerId::from_bytes([0u8; 32]), }; queues.add_pending_verify(xor_name_from_byte(0x01), entry); diff --git a/src/replication/config.rs b/src/replication/config.rs index 575008a7..dea3036f 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -40,6 +40,16 @@ pub const QUORUM_THRESHOLD: usize = 4; // floor(CLOSE_GROUP_SIZE / 2) + 1 /// Maximum number of closest nodes tracking paid status for a key. pub const PAID_LIST_CLOSE_GROUP_SIZE: usize = 20; +/// Number of furthest paid-list close-group peers treated as churny edge +/// voters. +/// +/// Once the paid-list close group reaches [`PAID_LIST_CLOSE_GROUP_SIZE`], edge +/// peers are queried, but a negative edge paid-list response does not count +/// into the paid-list majority denominator. A positive edge response does +/// count. This absorbs local routing-table disagreement at the boundary of the +/// paid close group. Undersized groups keep their ordinary strict majority. +pub const PAID_LIST_FLEX_EDGE_COUNT: usize = 4; + /// Number of closest peers to self eligible for neighbor sync. pub const NEIGHBOR_SYNC_SCOPE: usize = 20; @@ -157,6 +167,18 @@ pub const MAX_CONCURRENT_AUDIT_RESPONSES: usize = 16; /// comfortably covers the legitimate round-1 + round-2 overlap. pub const MAX_AUDIT_RESPONSES_PER_PEER: u32 = 2; +/// Maximum concurrent digest-only `AuditChallenge` responses from any single +/// source peer. +/// +/// Digest challenges are KB-scale replies: at most +/// `max_incoming_audit_keys(stored_chunks)` bounded disk reads plus BLAKE3 +/// digests. A higher per-source allowance absorbs the three legitimate issuer +/// subsystems (responsible-chunk audit, prune confirmation, and possession +/// checks) from one auditor without weakening the existing subtree/byte audit +/// budget. The multi-MiB subtree and byte challenge paths intentionally keep +/// [`MAX_AUDIT_RESPONSES_PER_PEER`] exactly unchanged. +pub const MAX_DIGEST_AUDIT_RESPONSES_PER_PEER: u32 = 8; + /// Concurrent fetches cap, derived from hardware thread count. /// /// Uses `std::thread::available_parallelism()` so the node scales to the @@ -300,6 +322,15 @@ const VERIFICATION_REQUEST_TIMEOUT_SECS: u64 = 15; pub const VERIFICATION_REQUEST_TIMEOUT: Duration = Duration::from_secs(VERIFICATION_REQUEST_TIMEOUT_SECS); +/// Maximum keys in one verification request/response batch. +/// +/// The 10 MiB replication wire cap is intentionally much higher because other +/// messages carry hint sets and chunk bytes. Verification requests do local +/// LMDB lookups per key on the responder's serial replication message path, so +/// this smaller cap bounds CPU/disk work and keeps honest large rounds +/// splittable instead of failing one oversized encode. +pub const MAX_VERIFICATION_KEYS_PER_REQUEST: usize = 1024; + /// Fetch request timeout. const FETCH_REQUEST_TIMEOUT_SECS: u64 = 30; /// Fetch request timeout. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 80776cdc..237d817b 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -16,6 +16,8 @@ pub mod admission; pub mod audit; +pub mod audit_coordinator; +pub(crate) mod audit_metrics; pub mod bootstrap; pub mod commitment; pub mod commitment_state; @@ -44,19 +46,25 @@ use crate::logging::{debug, error, info, warn}; use futures::stream::FuturesUnordered; use futures::{Future, StreamExt}; use rand::Rng; -use tokio::sync::{mpsc, Notify, RwLock, Semaphore}; +use tokio::sync::broadcast::error::RecvError; +use tokio::sync::{mpsc, Mutex, Notify, RwLock, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; use crate::ant_protocol::XorName; use crate::error::{Error, Result}; use crate::payment::{PaymentVerifier, VerificationContext}; use crate::replication::audit::AuditTickResult; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::AuditResponderClass; use crate::replication::commitment::{commitment_hash, StorageCommitment}; use crate::replication::commitment_state::{PeerCommitmentRecord, ResponderCommitmentState}; use crate::replication::config::{ max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, - MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, REPLICATION_PROTOCOL_ID, + MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, + MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, MAX_VERIFICATION_KEYS_PER_REQUEST, + REPLICATION_PROTOCOL_ID, }; use crate::replication::paid_list::PaidList; use crate::replication::protocol::{ @@ -81,6 +89,30 @@ use saorsa_core::{DhtNetworkEvent, P2PEvent, P2PNode, TrustEvent}; /// Prefix used by saorsa-core's request-response mechanism. const RR_PREFIX: &str = "/rr/"; +/// Bounded handoff from the P2P broadcast receiver to serial non-audit +/// replication processing. +/// +/// The receiver fast-paths digest `AuditChallenge`s immediately and queues +/// bulk/non-audit messages here. If this fills, the receiver handles the +/// message inline instead of dropping it, preserving delivery while bounding +/// memory. +const INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY: usize = 256; + +/// Maximum fresh-replication offers processed away from the serial +/// non-audit loop. +/// +/// Fresh offers can perform an on-chain payment verification and a 4 MiB LMDB +/// write. Four workers keep that latency off the responder dispatch path while +/// keeping concurrent EVM/storage pressure small and predictable. +const FRESH_OFFER_WORKER_LIMIT: usize = 4; + +/// Number of fixed keyed locks used to preserve fresh-offer ordering per key. +/// +/// A fixed shard set avoids unbounded per-key lock state. Same-key offers map +/// to the same shard and serialize; unrelated keys usually progress +/// independently under the worker bound. +const FRESH_OFFER_KEY_LOCK_SHARDS: usize = 64; + fn fresh_offer_payment_context() -> VerificationContext { VerificationContext::ClientPut } @@ -89,9 +121,23 @@ fn paid_notify_payment_context() -> VerificationContext { VerificationContext::PaidListAdmission } +fn new_fresh_offer_key_locks() -> FreshOfferKeyLocks { + Arc::new( + (0..FRESH_OFFER_KEY_LOCK_SHARDS) + .map(|_| Arc::new(Mutex::new(()))) + .collect(), + ) +} + +fn fresh_offer_key_lock_index(key: &XorName) -> usize { + usize::from(key[0]) % FRESH_OFFER_KEY_LOCK_SHARDS +} + /// Boxed future type for in-flight fetch tasks. type FetchFuture = Pin)> + Send>>; +type FreshOfferKeyLocks = Arc>>>; + /// Shared dependencies for one verification worker cycle. struct VerificationCycleContext<'a> { p2p_node: &'a Arc, @@ -130,6 +176,11 @@ const REPLICATION_TRUST_WEIGHT: f64 = 1.0; /// Bootstrap drain check interval in seconds. const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; +/// Grace period `shutdown()` waits for each background task (and, collectively, +/// the detached fresh-offer worker pool) to observe the cancellation token and +/// terminate before it gives up and aborts / abandons the wait. +const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); + /// How often the responder rebuilds + rotates its storage commitment. /// /// Each rebuild scans LMDB to compute leaf hashes; for ~10k keys this is @@ -295,6 +346,16 @@ pub struct ReplicationEngine { /// per-peer cap guarantees no single source can hold more than its share, /// so a flood self-throttles without denying service to everyone else. audit_responder_inflight: Arc>>, + /// Shared auditor-side limiter for outbound digest `AuditChallenge`s. + /// + /// Responsible-chunk audits, prune confirmations, and possession checks + /// all use this before sending so local bursts wait instead of breaching + /// the responder's deployed per-source admission cap. + audit_challenge_coordinator: Arc, + /// Bounded worker permits for expensive fresh-offer handling. + fresh_offer_worker_semaphore: Arc, + /// Fixed shard locks preserving per-key fresh-offer ordering. + fresh_offer_key_locks: FreshOfferKeyLocks, /// Receiver for fresh-write events from the chunk PUT handler. /// /// When present, `start()` spawns a drainer task that calls @@ -310,6 +371,11 @@ pub struct ReplicationEngine { shutdown: CancellationToken, /// Background task handles. task_handles: Vec>, + /// Tracks detached, short-lived fresh-offer worker tasks so `shutdown()` + /// can drain them. Unlike `task_handles` these are spawned on demand from + /// the message handler and hold `Arc` while writing, so they + /// must be awaited before the caller may reopen the LMDB environment. + worker_tracker: TaskTracker, } impl ReplicationEngine { @@ -367,11 +433,15 @@ impl ReplicationEngine { send_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_REPLICATION_SENDS)), audit_responder_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_AUDIT_RESPONSES)), audit_responder_inflight: Arc::new(RwLock::new(HashMap::new())), + audit_challenge_coordinator: Arc::new(AuditChallengeCoordinator::new()), + fresh_offer_worker_semaphore: Arc::new(Semaphore::new(FRESH_OFFER_WORKER_LIMIT)), + fresh_offer_key_locks: new_fresh_offer_key_locks(), fresh_write_rx: Some(fresh_write_rx), possession_check_tx, possession_check_rx: Some(possession_check_rx), shutdown, task_handles: Vec::new(), + worker_tracker: TaskTracker::new(), }) } @@ -487,6 +557,7 @@ impl ReplicationEngine { &self.storage, &self.config, &self.sync_state, + &self.audit_challenge_coordinator, &self.shutdown, ) .await; @@ -563,16 +634,35 @@ impl ReplicationEngine { pub async fn shutdown(&mut self) { self.shutdown.cancel(); for (i, mut handle) in self.task_handles.drain(..).enumerate() { - match tokio::time::timeout(std::time::Duration::from_secs(10), &mut handle).await { + match tokio::time::timeout(SHUTDOWN_TASK_DRAIN_TIMEOUT, &mut handle).await { Ok(Ok(())) => {} Ok(Err(e)) if e.is_cancelled() => {} Ok(Err(e)) => warn!("Replication task {i} panicked during shutdown: {e}"), Err(_) => { - warn!("Replication task {i} did not stop within 10s, aborting"); + warn!( + "Replication task {i} did not stop within {}s, aborting", + SHUTDOWN_TASK_DRAIN_TIMEOUT.as_secs() + ); handle.abort(); } } } + + // Drain detached fresh-offer workers. The serial message handler has + // already stopped (its handle is drained above), so no new workers can + // be spawned; each observes the cancelled token and finishes promptly, + // releasing its `Arc` before this returns. + self.worker_tracker.close(); + if tokio::time::timeout(SHUTDOWN_TASK_DRAIN_TIMEOUT, self.worker_tracker.wait()) + .await + .is_err() + { + warn!( + "Fresh-offer workers did not drain within {}s; {} still in flight", + SHUTDOWN_TASK_DRAIN_TIMEOUT.as_secs(), + self.worker_tracker.len() + ); + } } /// Trigger an early neighbor sync round. @@ -672,6 +762,7 @@ impl ReplicationEngine { let storage = Arc::clone(&self.storage); let config = Arc::clone(&self.config); let sync_state = Arc::clone(&self.sync_state); + let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); let shutdown = self.shutdown.clone(); let handle = tokio::spawn(async move { @@ -687,6 +778,7 @@ impl ReplicationEngine { let storage = Arc::clone(&storage); let config = Arc::clone(&config); let sync_state = Arc::clone(&sync_state); + let audit_challenge_coordinator = Arc::clone(&audit_challenge_coordinator); let shutdown = shutdown.clone(); let delay_min = config.possession_check_delay_min; let delay_max = config.possession_check_delay_max; @@ -702,6 +794,7 @@ impl ReplicationEngine { &storage, &config, &sync_state, + &audit_challenge_coordinator, &shutdown, ) .await; @@ -742,6 +835,9 @@ impl ReplicationEngine { let sync_state = Arc::clone(&self.sync_state); let audit_responder_semaphore = Arc::clone(&self.audit_responder_semaphore); let audit_responder_inflight = Arc::clone(&self.audit_responder_inflight); + let fresh_offer_worker_semaphore = Arc::clone(&self.fresh_offer_worker_semaphore); + let fresh_offer_key_locks = Arc::clone(&self.fresh_offer_key_locks); + let worker_tracker = self.worker_tracker.clone(); // ADR-0002 gossip-audit trigger: bundled state so an ingested *changed* // commitment can spawn a probabilistic, cooldown-gated subtree audit. @@ -753,61 +849,156 @@ impl ReplicationEngine { cooldown: Arc::clone(&audit_on_gossip_cooldown), }; + let handler_context = ReplicationMessageHandlerContext { + p2p_node: Arc::clone(&p2p), + storage, + paid_list, + payment_verifier, + queues, + config: Arc::clone(&config), + is_bootstrapping, + bootstrap_state, + sync_history, + sync_cycle_epoch, + repair_proofs: Arc::clone(&repair_proofs), + last_commitment_by_peer: Arc::clone(&last_commitment_by_peer), + ever_capable_peers, + sig_verify_attempts: Arc::clone(&sig_verify_attempts), + my_commitment_state, + gossip_audit, + audit_responder_semaphore, + audit_responder_inflight, + fresh_offer_worker_semaphore, + fresh_offer_key_locks, + shutdown: shutdown.clone(), + worker_tracker, + }; + + let (replication_tx, mut replication_rx) = + mpsc::channel::(INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY); + let serial_context = handler_context.clone(); + let serial_shutdown = shutdown.clone(); + let serial_handle = tokio::spawn(async move { + loop { + tokio::select! { + () = serial_shutdown.cancelled() => break, + inbound = replication_rx.recv() => { + let Some(inbound) = inbound else { break }; + let source = inbound.source; + match handle_replication_message( + &source, + inbound.msg, + &serial_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await + { + Ok(()) => {} + Err(e) => { + debug!("Replication message from {source} error: {e}"); + } + } + } + } + } + debug!("Replication non-audit serial handler shut down"); + }); + self.task_handles.push(serial_handle); + let handle = tokio::spawn(async move { loop { tokio::select! { () = shutdown.cancelled() => break, event = p2p_events.recv() => { - let Ok(event) = event else { continue }; - if let P2PEvent::Message { - topic, - source: Some(source), - data, - .. - } = event { - // Determine if this is a replication message - // and whether it arrived via the /rr/ request-response - // path (which wraps payloads in RequestResponseEnvelope). - let rr_info = if topic == REPLICATION_PROTOCOL_ID { - Some((data.clone(), None)) - } else if topic.starts_with(RR_PREFIX) - && &topic[RR_PREFIX.len()..] == REPLICATION_PROTOCOL_ID + let event = match event { + Ok(event) => event, + Err(error) => { + handle_replication_event_recv_error(&error); + continue; + } + }; + let Some((source, payload, rr_message_id)) = + replication_payload_from_event(event) + else { + continue; + }; + let received_at = Instant::now(); + let msg = match ReplicationMessage::decode(&payload) { + Ok(msg) => msg, + Err(e) => { + debug!("Replication message from {source} decode error: {e}"); + continue; + } + }; + let inbound = InboundReplicationMessage { + source, + msg, + rr_message_id, + received_at, + }; + if matches!( + inbound.msg.body, + ReplicationMessageBody::AuditChallenge(_) + ) { + let source = inbound.source; + match handle_replication_message( + &source, + inbound.msg, + &handler_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await { - P2PNode::parse_request_envelope(&data) - .filter(|(_, is_resp, _)| !is_resp) - .map(|(msg_id, _, payload)| (payload, Some(msg_id))) - } else { - None - }; - if let Some((payload, rr_message_id)) = rr_info { + Ok(()) => {} + Err(e) => { + debug!("Replication message from {source} error: {e}"); + } + } + continue; + } + match replication_tx.try_send(inbound) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(inbound)) => { + warn!( + "Replication non-audit queue full; handling message from {} inline", + inbound.source + ); + let source = inbound.source; match handle_replication_message( &source, - &payload, - &p2p, - &storage, - &paid_list, - &payment_verifier, - &queues, - &config, - &is_bootstrapping, - &bootstrap_state, - &sync_history, - &sync_cycle_epoch, - &repair_proofs, - &last_commitment_by_peer, - &ever_capable_peers, - &sig_verify_attempts, - &my_commitment_state, - &gossip_audit, - &audit_responder_semaphore, - &audit_responder_inflight, - rr_message_id.as_deref(), - ).await { + inbound.msg, + &handler_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await + { + Ok(()) => {} + Err(e) => { + debug!("Replication message from {source} error: {e}"); + } + } + } + Err(mpsc::error::TrySendError::Closed(inbound)) => { + warn!( + "Replication non-audit queue closed; handling message from {} inline", + inbound.source + ); + let source = inbound.source; + match handle_replication_message( + &source, + inbound.msg, + &handler_context, + inbound.received_at, + inbound.rr_message_id.as_deref(), + ) + .await + { Ok(()) => {} Err(e) => { - debug!( - "Replication message from {source} error: {e}" - ); + debug!("Replication message from {source} error: {e}"); } } } @@ -821,7 +1012,43 @@ impl ReplicationEngine { // previous approach of checking every PeerConnected / // PeerDisconnected event against the close group. dht_event = dht_events.recv() => { - let Ok(dht_event) = dht_event else { continue }; + let dht_event = match dht_event { + Ok(event) => event, + Err(RecvError::Lagged(missed)) => { + // Under heavy churn the broadcast buffer can overflow + // and drop routing-table events — the moment + // convergence matters most. A dropped + // KClosestPeersChanged means its entrants were never + // queued, so draining priority_order cannot recover + // them. Resync from ground truth instead: snapshot the + // current close-peer set and queue every member. Dedup + // (queue_priority_peers) and per-peer cooldown + // (select_next_sync_peer) drop peers already queued or + // recently synced, so only genuine entrants surface. + warn!( + "Missed {missed} DHT routing events (broadcast lag); resynchronizing close-peer set for neighbor sync" + ); + let self_id = *p2p.peer_id(); + let neighbors = neighbor_sync::snapshot_close_neighbors( + &p2p, + &self_id, + config.neighbor_sync_scope, + ) + .await; + let requeued = { + let mut state = sync_state.write().await; + state.queue_priority_peers(neighbors) + }; + if requeued > 0 { + debug!( + "Resync after broadcast lag queued {requeued} close peers for priority neighbor sync" + ); + sync_trigger.notify_one(); + } + continue; + } + Err(RecvError::Closed) => continue, + }; match dht_event { DhtNetworkEvent::KClosestPeersChanged { old, new } => { let old_peers = old @@ -908,6 +1135,7 @@ impl ReplicationEngine { let last_commitment_by_peer = Arc::clone(&self.last_commitment_by_peer); let ever_capable_peers = Arc::clone(&self.ever_capable_peers); let sig_verify_attempts = Arc::clone(&self.sig_verify_attempts); + let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); // ADR-0002: a peer's commitment also arrives on the sync RESPONSE path // (we initiated, they piggybacked theirs). Carry a gossip-audit trigger // here too so a peer that only ever answers — never initiates sync — @@ -922,12 +1150,22 @@ impl ReplicationEngine { let handle = tokio::spawn(async move { loop { - let interval = config.random_neighbor_sync_interval(); - tokio::select! { - () = shutdown.cancelled() => break, - () = tokio::time::sleep(interval) => {} - () = sync_trigger.notified() => { - debug!("Neighbor sync triggered by topology change"); + // Park for the periodic tick or an explicit trigger ONLY when no + // priority (topology-change) peers are queued. `sync_trigger` is a + // coalescing `Notify`, so a churn burst that queues many entrants + // produces a single wakeup; parking after draining one batch would + // leave the rest waiting up to a full periodic tick. `priority_order` + // is the durable record of pending work, so drain it back-to-back — + // each round removes the peers it selects (`select_next_sync_peer`), + // so the drain terminates once the queue empties. + if !sync_state.read().await.has_priority_peers() { + let interval = config.random_neighbor_sync_interval(); + tokio::select! { + () = shutdown.cancelled() => break, + () = tokio::time::sleep(interval) => {} + () = sync_trigger.notified() => { + debug!("Neighbor sync triggered by topology change"); + } } } // Wrap the sync round in a select so shutdown cancels @@ -951,6 +1189,7 @@ impl ReplicationEngine { &last_commitment_by_peer, &ever_capable_peers, &sig_verify_attempts, + &audit_challenge_coordinator, &gossip_audit, ) => {} } @@ -999,6 +1238,7 @@ impl ReplicationEngine { let bootstrap_state = Arc::clone(&self.bootstrap_state); let is_bootstrapping = Arc::clone(&self.is_bootstrapping); let sync_state = Arc::clone(&self.sync_state); + let audit_challenge_coordinator = Arc::clone(&self.audit_challenge_coordinator); let handle = tokio::spawn(async move { // Invariant 19: wait for bootstrap to drain before starting audits. @@ -1027,6 +1267,7 @@ impl ReplicationEngine { &config, &history, &repair_proofs, + &audit_challenge_coordinator, current_sync_epoch, bootstrapping, ) @@ -1051,6 +1292,7 @@ impl ReplicationEngine { &config, &history, &repair_proofs, + &audit_challenge_coordinator, current_sync_epoch, bootstrapping, ) @@ -1174,20 +1416,24 @@ impl ReplicationEngine { let Some(candidate) = q.dequeue_fetch() else { break; }; + let fetch_key = candidate.key; let Some(&source) = candidate.sources.first() else { warn!( - "Fetch candidate {} has no sources — dropping", - hex::encode(candidate.key) + "Fetch candidate {} has no sources; requeueing for verification", + hex::encode(fetch_key) + ); + let _ = q.requeue_candidate_for_verification( + candidate, + config.verification_request_timeout, ); continue; }; - q.start_fetch(candidate.key, source, candidate.sources.clone()); + q.start_dequeued_fetch(candidate, source); let p2p = Arc::clone(&p2p); let storage = Arc::clone(&storage); let config = Arc::clone(&config); let token = shutdown.clone(); - let fetch_key = candidate.key; in_flight.push(Box::pin(async move { let handle = tokio::spawn(async move { // Cancel-aware: abort when the engine shuts down. @@ -1269,15 +1515,20 @@ impl ReplicationEngine { })); false } else { - q.complete_fetch(&key); - true + !q.requeue_fetch_for_verification( + &key, + config.verification_request_timeout, + ) } } } } else { - // Task panicked — reclaim the in-flight slot. - q.complete_fetch(&key); - true + // Task panicked — retry verification when this was + // a verified repair, otherwise reclaim the slot. + !q.requeue_fetch_for_verification( + &key, + config.verification_request_timeout, + ) }; // Shrink bootstrap pending set on terminal exit. @@ -1561,6 +1812,89 @@ struct AuditResponderGuard { peer: PeerId, } +#[derive(Clone)] +struct ReplicationMessageHandlerContext { + p2p_node: Arc, + storage: Arc, + paid_list: Arc, + payment_verifier: Arc, + queues: Arc>, + config: Arc, + is_bootstrapping: Arc>, + bootstrap_state: Arc>, + sync_history: Arc>>, + sync_cycle_epoch: Arc>, + repair_proofs: Arc>, + last_commitment_by_peer: Arc>>, + ever_capable_peers: Arc>>, + sig_verify_attempts: Arc>>, + my_commitment_state: Arc, + gossip_audit: GossipAuditTrigger, + audit_responder_semaphore: Arc, + audit_responder_inflight: Arc>>, + fresh_offer_worker_semaphore: Arc, + fresh_offer_key_locks: FreshOfferKeyLocks, + /// Cancellation token so detached fresh-offer workers abort in-flight work + /// (payment verification, LMDB writes) when the engine shuts down. + shutdown: CancellationToken, + /// Tracker the detached fresh-offer workers register with so `shutdown()` + /// can await their completion and the release of their `Arc`. + worker_tracker: TaskTracker, +} + +struct InboundReplicationMessage { + source: PeerId, + msg: ReplicationMessage, + rr_message_id: Option, + received_at: Instant, +} + +impl AuditResponderClass { + const fn per_peer_limit(self) -> u32 { + match self { + Self::Digest => MAX_DIGEST_AUDIT_RESPONSES_PER_PEER, + Self::Subtree | Self::Byte => MAX_AUDIT_RESPONSES_PER_PEER, + } + } +} + +fn handle_replication_event_recv_error(error: &RecvError) { + match error { + RecvError::Lagged(missed) => { + audit_metrics::record_replication_event_lagged(*missed); + warn!( + "Missed {missed} P2P events on replication branch (broadcast lag); \ + replication messages may have been dropped before dispatch" + ); + } + RecvError::Closed => { + warn!("P2P event stream closed on replication branch"); + } + } +} + +fn replication_payload_from_event(event: P2PEvent) -> Option<(PeerId, Vec, Option)> { + let P2PEvent::Message { + topic, + source: Some(source), + data, + .. + } = event + else { + return None; + }; + + if topic == REPLICATION_PROTOCOL_ID { + return Some((source, data, None)); + } + if topic.starts_with(RR_PREFIX) && &topic[RR_PREFIX.len()..] == REPLICATION_PROTOCOL_ID { + return P2PNode::parse_request_envelope(&data) + .filter(|(_, is_resp, _)| !is_resp) + .map(|(msg_id, _, payload)| (source, payload, Some(msg_id))); + } + None +} + impl Drop for AuditResponderGuard { fn drop(&mut self) { // Decrement (and prune to keep the map bounded) without blocking the @@ -1605,6 +1939,7 @@ async fn admit_audit_responder( semaphore: &Arc, inflight: &Arc>>, source: &PeerId, + class: AuditResponderClass, ) -> Option { // Per-peer cap first (cheap, and the fairness-critical bound), committed // under the write lock so concurrent challenges from the same peer can't @@ -1612,7 +1947,7 @@ async fn admit_audit_responder( { let mut map = inflight.write().await; let entry = map.entry(*source).or_insert(0); - if *entry >= MAX_AUDIT_RESPONSES_PER_PEER { + if *entry >= class.per_peer_limit() { return None; } *entry += 1; @@ -1641,61 +1976,31 @@ async fn admit_audit_responder( /// When `rr_message_id` is `Some`, the request arrived via the `/rr/` /// request-response path and the response must be sent via `send_response` /// so saorsa-core can route it back to the waiting `send_request` caller. -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +#[allow(clippy::too_many_lines)] async fn handle_replication_message( source: &PeerId, - data: &[u8], - p2p_node: &Arc, - storage: &Arc, - paid_list: &Arc, - payment_verifier: &Arc, - queues: &Arc>, - config: &ReplicationConfig, - is_bootstrapping: &Arc>, - bootstrap_state: &Arc>, - sync_history: &Arc>>, - sync_cycle_epoch: &Arc>, - repair_proofs: &Arc>, - last_commitment_by_peer: &Arc>>, - ever_capable_peers: &Arc>>, - sig_verify_attempts: &Arc>>, - my_commitment_state: &Arc, - gossip_audit: &GossipAuditTrigger, - audit_responder_semaphore: &Arc, - audit_responder_inflight: &Arc>>, + msg: ReplicationMessage, + ctx: &ReplicationMessageHandlerContext, + received_at: Instant, rr_message_id: Option<&str>, ) -> Result<()> { - let msg = ReplicationMessage::decode(data) - .map_err(|e| Error::Protocol(format!("Failed to decode replication message: {e}")))?; - match msg.body { - ReplicationMessageBody::FreshReplicationOffer(ref offer) => { - handle_fresh_offer( - source, - offer, - storage, - paid_list, - payment_verifier, - p2p_node, - config, - msg.request_id, - rr_message_id, - ) - .await + ReplicationMessageBody::FreshReplicationOffer(offer) => { + dispatch_fresh_offer(*source, offer, ctx, msg.request_id, rr_message_id).await } ReplicationMessageBody::PaidNotify(ref notify) => { handle_paid_notify( source, notify, - paid_list, - payment_verifier, - p2p_node, - config, + &ctx.paid_list, + &ctx.payment_verifier, + &ctx.p2p_node, + &ctx.config, ) .await } ReplicationMessageBody::NeighborSyncRequest(ref request) => { - let bootstrapping = *is_bootstrapping.read().await; + let bootstrapping = *ctx.is_bootstrapping.read().await; // Phase-3 storage-bound audit: store the sender's // commitment for use as `expected_commitment_hash` in // future audits. Verify signature before storing so a peer @@ -1703,31 +2008,31 @@ async fn handle_replication_message( if let Some(target) = ingest_peer_commitment( source, request.commitment.as_ref(), - p2p_node, - last_commitment_by_peer, - ever_capable_peers, - sig_verify_attempts, + &ctx.p2p_node, + &ctx.last_commitment_by_peer, + &ctx.ever_capable_peers, + &ctx.sig_verify_attempts, ) .await { - maybe_trigger_gossip_audit(gossip_audit, source, target).await; + maybe_trigger_gossip_audit(&ctx.gossip_audit, source, target).await; } handle_neighbor_sync_request( source, request, - p2p_node, - storage, - paid_list, - queues, - config, + &ctx.p2p_node, + &ctx.storage, + &ctx.paid_list, + &ctx.queues, + &ctx.config, bootstrapping, - bootstrap_state, - sync_history, - sync_cycle_epoch, - repair_proofs, + &ctx.bootstrap_state, + &ctx.sync_history, + &ctx.sync_cycle_epoch, + &ctx.repair_proofs, // Atomically snapshot + mark-gossiped: emitted in the sync // response, so we must stay answerable for it (ADR-0002). - my_commitment_state + ctx.my_commitment_state .current_for_gossip() .map(|b| b.commitment().clone()), msg.request_id, @@ -1739,9 +2044,9 @@ async fn handle_replication_message( handle_verification_request( source, request, - storage, - paid_list, - p2p_node, + &ctx.storage, + &ctx.paid_list, + &ctx.p2p_node, msg.request_id, rr_message_id, ) @@ -1751,8 +2056,8 @@ async fn handle_replication_message( handle_fetch_request( source, request, - storage, - p2p_node, + &ctx.storage, + &ctx.p2p_node, msg.request_id, rr_message_id, ) @@ -1773,19 +2078,33 @@ async fn handle_replication_message( // is hit. Responsible/prune audit timeouts are penalised by the // caller, so the caps must remain high enough for honest audit load; // the per-peer share still prevents one flooder from starving others. - let Some(guard) = - admit_audit_responder(audit_responder_semaphore, audit_responder_inflight, source) - .await + let Some(guard) = admit_audit_responder( + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, + source, + AuditResponderClass::Digest, + ) + .await else { + audit_metrics::record_admission_drop(AuditResponderClass::Digest); warn!( "Audit challenge reply not sent: kind=responsible response=dropped \ - source={source} (audit-responder capacity reached)" + source={source} responder_class={} (audit-responder capacity reached)", + AuditResponderClass::Digest.as_str(), ); return Ok(()); }; - let bootstrapping = *is_bootstrapping.read().await; - let storage = Arc::clone(storage); - let p2p_node = Arc::clone(p2p_node); + let bootstrapping = *ctx.is_bootstrapping.read().await; + let dispatch_latency = received_at.elapsed(); + audit_metrics::record_digest_dispatch_latency(dispatch_latency); + debug!( + audit_type = "digest_responder", + dispatch_latency_ms = dispatch_latency.as_millis(), + source = %source, + "Audit challenge dispatch latency measured" + ); + let storage = Arc::clone(&ctx.storage); + let p2p_node = Arc::clone(&ctx.p2p_node); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -1824,20 +2143,26 @@ async fn handle_replication_message( "Audit challenge received: kind=subtree source={source} request_response={}", rr_message_id.is_some(), ); - let Some(guard) = - admit_audit_responder(audit_responder_semaphore, audit_responder_inflight, source) - .await + let Some(guard) = admit_audit_responder( + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, + source, + AuditResponderClass::Subtree, + ) + .await else { + audit_metrics::record_admission_drop(AuditResponderClass::Subtree); warn!( "Audit challenge reply not sent: kind=subtree response=dropped \ - source={source} (audit-responder capacity reached)" + source={source} responder_class={} (audit-responder capacity reached)", + AuditResponderClass::Subtree.as_str(), ); return Ok(()); }; - let bootstrapping = *is_bootstrapping.read().await; - let storage = Arc::clone(storage); - let p2p_node = Arc::clone(p2p_node); - let my_commitment_state = Arc::clone(my_commitment_state); + let bootstrapping = *ctx.is_bootstrapping.read().await; + let storage = Arc::clone(&ctx.storage); + let p2p_node = Arc::clone(&ctx.p2p_node); + let my_commitment_state = Arc::clone(&ctx.my_commitment_state); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -1886,20 +2211,26 @@ async fn handle_replication_message( "Audit challenge received: kind=byte source={source} request_response={}", rr_message_id.is_some(), ); - let Some(guard) = - admit_audit_responder(audit_responder_semaphore, audit_responder_inflight, source) - .await + let Some(guard) = admit_audit_responder( + &ctx.audit_responder_semaphore, + &ctx.audit_responder_inflight, + source, + AuditResponderClass::Byte, + ) + .await else { + audit_metrics::record_admission_drop(AuditResponderClass::Byte); warn!( "Audit challenge reply not sent: kind=byte response=dropped \ - source={source} (audit-responder capacity reached)" + source={source} responder_class={} (audit-responder capacity reached)", + AuditResponderClass::Byte.as_str(), ); return Ok(()); }; - let bootstrapping = *is_bootstrapping.read().await; - let storage = Arc::clone(storage); - let p2p_node = Arc::clone(p2p_node); - let my_commitment_state = Arc::clone(my_commitment_state); + let bootstrapping = *ctx.is_bootstrapping.read().await; + let storage = Arc::clone(&ctx.storage); + let p2p_node = Arc::clone(&ctx.p2p_node); + let my_commitment_state = Arc::clone(&ctx.my_commitment_state); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); @@ -1953,6 +2284,83 @@ async fn handle_replication_message( // Per-message-type handlers // --------------------------------------------------------------------------- +async fn dispatch_fresh_offer( + source: PeerId, + offer: protocol::FreshReplicationOffer, + ctx: &ReplicationMessageHandlerContext, + request_id: u64, + rr_message_id: Option<&str>, +) -> Result<()> { + let rr_message_id = rr_message_id.map(ToOwned::to_owned); + let permit = Arc::clone(&ctx.fresh_offer_worker_semaphore).try_acquire_owned(); + let Ok(permit) = permit else { + debug!( + "Fresh-offer worker pool saturated; handling offer for {} from {source} inline", + hex::encode(offer.key) + ); + return handle_fresh_offer_serialized( + &source, + &offer, + ctx, + request_id, + rr_message_id.as_deref(), + ) + .await; + }; + + let ctx = ctx.clone(); + let tracker = ctx.worker_tracker.clone(); + let shutdown = ctx.shutdown.clone(); + // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds + // an `Arc` while writing, and the shutdown contract requires + // those references be released before the caller reopens the environment. + // The `select!` lets it abandon in-flight work promptly on cancellation + // instead of blocking shutdown for the full drain grace period. + tracker.spawn(async move { + let _permit = permit; + tokio::select! { + () = shutdown.cancelled() => { + debug!("Fresh-offer worker for {source} cancelled during shutdown"); + } + result = handle_fresh_offer_serialized( + &source, + &offer, + &ctx, + request_id, + rr_message_id.as_deref(), + ) => { + if let Err(e) = result { + debug!("Fresh replication offer from {source} error: {e}"); + } + } + } + }); + Ok(()) +} + +async fn handle_fresh_offer_serialized( + source: &PeerId, + offer: &protocol::FreshReplicationOffer, + ctx: &ReplicationMessageHandlerContext, + request_id: u64, + rr_message_id: Option<&str>, +) -> Result<()> { + let lock_index = fresh_offer_key_lock_index(&offer.key); + let _key_guard = ctx.fresh_offer_key_locks[lock_index].lock().await; + handle_fresh_offer( + source, + offer, + &ctx.storage, + &ctx.paid_list, + &ctx.payment_verifier, + &ctx.p2p_node, + &ctx.config, + request_id, + rr_message_id, + ) + .await +} + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn handle_fresh_offer( source: &PeerId, @@ -2385,13 +2793,39 @@ async fn handle_verification_request( request_id: u64, rr_message_id: Option<&str>, ) -> Result<()> { - // No per-request key count limit: the wire message size limit - // (MAX_REPLICATION_MESSAGE_SIZE) already caps the payload. Verification - // does cheap storage lookups per key, not expensive computation like - // audit digest generation. + #[derive(Clone, Copy)] + enum CachedPaidLookup { + NotChecked, + Checked(Option), + } - #[allow(clippy::cast_possible_truncation)] - let keys_len = request.keys.len() as u32; + #[derive(Clone, Copy)] + struct CachedVerificationLookup { + present: Option, + paid: CachedPaidLookup, + } + + let requested_keys = if request.keys.len() > MAX_VERIFICATION_KEYS_PER_REQUEST { + warn!( + "Verification request from {source} has {} keys, exceeding max {MAX_VERIFICATION_KEYS_PER_REQUEST}; answering capped prefix", + request.keys.len(), + ); + &request.keys[..MAX_VERIFICATION_KEYS_PER_REQUEST] + } else { + request.keys.as_slice() + }; + + if request.paid_list_check_indices.len() > request.keys.len() { + warn!( + "Verification request from {source} has {} paid-list indices for {} keys; rejecting batch", + request.paid_list_check_indices.len(), + request.keys.len(), + ); + send_verification_results(source, p2p_node, request_id, Vec::new(), rr_message_id).await; + return Ok(()); + } + + let keys_len = u32::try_from(requested_keys.len()).unwrap_or(u32::MAX); let paid_check_set: HashSet = request .paid_list_check_indices .iter() @@ -2400,7 +2834,7 @@ async fn handle_verification_request( if idx >= keys_len { warn!( "Verification request from {source}: paid_list_check_index {idx} out of bounds (keys.len() = {})", - request.keys.len(), + requested_keys.len(), ); false } else { @@ -2409,21 +2843,72 @@ async fn handle_verification_request( }) .collect(); - let mut results = Vec::with_capacity(request.keys.len()); - for (i, key) in request.keys.iter().enumerate() { - let present = storage.exists(key).unwrap_or(false); - let paid = if paid_check_set.contains(&u32::try_from(i).unwrap_or(u32::MAX)) { - Some(paid_list.contains(key).unwrap_or(false)) + let mut results = Vec::with_capacity(requested_keys.len()); + let mut lookup_cache: HashMap = HashMap::new(); + for (i, key) in requested_keys.iter().enumerate() { + let needs_paid = paid_check_set.contains(&u32::try_from(i).unwrap_or(u32::MAX)); + let cached = lookup_cache.entry(*key).or_insert_with(|| { + let present = match storage.exists(key) { + Ok(present) => Some(present), + Err(e) => { + warn!( + "Verification request from {source}: failed to check storage for {}: {e}", + hex::encode(key) + ); + None + } + }; + CachedVerificationLookup { + present, + paid: CachedPaidLookup::NotChecked, + } + }); + + if needs_paid && matches!(cached.paid, CachedPaidLookup::NotChecked) { + cached.paid = CachedPaidLookup::Checked(match paid_list.contains(key) { + Ok(paid) => Some(paid), + Err(e) => { + warn!( + "Verification request from {source}: failed to check paid-list for {}: {e}", + hex::encode(key) + ); + None + } + }); + } + + let paid = if needs_paid { + match cached.paid { + CachedPaidLookup::Checked(paid) => paid, + CachedPaidLookup::NotChecked => None, + } } else { None }; + + if cached.present.is_none() && paid.is_none() { + continue; + } + results.push(protocol::KeyVerificationResult { key: *key, - present, + present: cached.present.unwrap_or(false), paid, }); } + send_verification_results(source, p2p_node, request_id, results, rr_message_id).await; + + Ok(()) +} + +async fn send_verification_results( + source: &PeerId, + p2p_node: &Arc, + request_id: u64, + results: Vec, + rr_message_id: Option<&str>, +) { send_replication_response( source, p2p_node, @@ -2432,8 +2917,6 @@ async fn handle_verification_request( rr_message_id, ) .await; - - Ok(()) } async fn handle_fetch_request( @@ -2654,6 +3137,7 @@ async fn run_neighbor_sync_round( last_commitment_by_peer: &Arc>>, ever_capable_peers: &Arc>>, sig_verify_attempts: &Arc>>, + audit_challenge_coordinator: &Arc, gossip_audit: &GossipAuditTrigger, ) { let self_id = *p2p_node.peer_id(); @@ -2696,6 +3180,7 @@ async fn run_neighbor_sync_round( repair_proof_now: None, allow_remote_prune_audits, commitment_state: Some(commitment_state), + audit_challenge_coordinator, }) .await; @@ -3046,6 +3531,7 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, + next_verify_at: now, hint_sender: *source_peer, }, ); @@ -3070,6 +3556,7 @@ async fn admit_and_queue_hints( verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: now, + next_verify_at: now, hint_sender: *source_peer, }, ); @@ -3121,14 +3608,24 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // Evict stale entries that have been pending too long (e.g. unreachable // verification targets during a network partition). - { + let stale_pending_keys = { let mut q = queues.write().await; - q.evict_stale(config::PENDING_VERIFY_MAX_AGE); + q.evict_stale(config::PENDING_VERIFY_MAX_AGE) + }; + if !stale_pending_keys.is_empty() { + update_bootstrap_after_verification( + &stale_pending_keys, + bootstrap_state, + queues, + is_bootstrapping, + bootstrap_complete_notify, + ) + .await; } let pending_keys = { let q = queues.read().await; - q.pending_keys() + q.ready_pending_keys(Instant::now()) }; if pending_keys.is_empty() { @@ -3231,17 +3728,21 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { terminal_keys.push(key); continue; } - let sources = evidence.get(&key).map_or_else(Vec::new, |ev| { + let mut sources = evidence.get(&key).map_or_else(Vec::new, |ev| { quorum::present_sources_for_key(&key, ev, &targets) }); + let replica_hint_sender = q.get_pending(&key).and_then(|entry| { + (entry.pipeline == HintPipeline::Replica).then_some(entry.hint_sender) + }); + if let Some(hint_sender) = replica_hint_sender { + add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + } if sources.is_empty() { - // Terminal failure: remove pending and report. No fetch path. - q.remove_pending(&key); warn!( - "Locally paid key {} has no responding holders (possible data loss)", + "Locally paid key {} has no responding holders yet; deferring retry", hex::encode(key) ); - terminal_keys.push(key); + q.defer_pending(&key, config.verification_request_timeout); } else { let distance = crate::client::xor_distance(&key, p2p_node.peer_id().as_bytes()); // Atomic remove+enqueue: if fetch_queue is at capacity, the @@ -3339,7 +3840,8 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { provers_snapshot.is_credited_holder(key, peer, hash) }; - let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline)> = Vec::new(); + let mut evaluated: Vec<(XorName, KeyVerificationOutcome, HintPipeline, PeerId)> = + Vec::new(); { let q = queues.read().await; for key in &keys_needing_network { @@ -3356,13 +3858,13 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { config, holder_credit, ); - evaluated.push((*key, outcome, entry.pipeline)); + evaluated.push((*key, outcome, entry.pipeline, entry.hint_sender)); } } // read lock released // Step 4: Insert verified keys into PaidForList (no lock held). let mut paid_insert_keys: Vec = Vec::new(); - for (key, outcome, _) in &evaluated { + for (key, outcome, _, _) in &evaluated { if matches!( outcome, KeyVerificationOutcome::QuorumVerified { .. } @@ -3382,7 +3884,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // paid-only hint can safely repair a missing replica using sources // from the same verification round. let mut paid_only_fetch_keys: HashSet = HashSet::new(); - for (key, outcome, pipeline) in &evaluated { + for (key, outcome, pipeline, _) in &evaluated { if *pipeline == HintPipeline::PaidOnly && matches!( outcome, @@ -3404,38 +3906,42 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { // Step 5: Update queues with the evaluated outcomes. let mut q = queues.write().await; - for (key, outcome, pipeline) in evaluated { + for (key, outcome, pipeline, hint_sender) in evaluated { match outcome { KeyVerificationOutcome::QuorumVerified { sources } | KeyVerificationOutcome::PaidListVerified { sources } => { + let mut fetch_sources = sources; + add_replica_hint_sender_source(&mut fetch_sources, pipeline, hint_sender); let fetch_eligible = pipeline == HintPipeline::Replica || paid_only_fetch_keys.contains(&key); - if fetch_eligible && !sources.is_empty() { + if fetch_eligible && !fetch_sources.is_empty() { let distance = crate::client::xor_distance(&key, p2p_node.peer_id().as_bytes()); // Atomic remove+enqueue: on fetch_queue capacity miss // the pending entry is preserved so this verified key // is retried on the next cycle (no silent drop). - let _ = q.promote_pending_to_fetch(key, distance, sources); + let _ = q.promote_pending_to_fetch(key, distance, fetch_sources); // Not terminal — either moved to fetch queue, or // retained as pending until queue drains. - } else if fetch_eligible && sources.is_empty() { + } else if fetch_eligible && fetch_sources.is_empty() { warn!( - "Verified storage-admitted key {} has no holders (possible data loss)", + "Verified storage-admitted key {} has no holders yet; deferring retry", hex::encode(key) ); - q.remove_pending(&key); - terminal_keys.push(key); + q.defer_pending(&key, config.verification_request_timeout); } else { q.remove_pending(&key); terminal_keys.push(key); } } - KeyVerificationOutcome::QuorumFailed - | KeyVerificationOutcome::QuorumInconclusive => { + KeyVerificationOutcome::QuorumFailed => { q.remove_pending(&key); terminal_keys.push(key); } + KeyVerificationOutcome::QuorumInconclusive => { + q.set_pending_state(&key, VerificationState::QuorumInconclusive); + q.defer_pending(&key, config.verification_request_timeout); + } } } } @@ -3475,6 +3981,16 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } } +fn add_replica_hint_sender_source( + sources: &mut Vec, + pipeline: HintPipeline, + hint_sender: PeerId, +) { + if pipeline == HintPipeline::Replica && !sources.contains(&hint_sender) { + sources.push(hint_sender); + } +} + /// Post-verification bootstrap bookkeeping: remove terminal keys from the /// bootstrap pending set and transition out of bootstrapping when drained. async fn update_bootstrap_after_verification( @@ -3828,7 +4344,10 @@ async fn handle_subtree_audit_result( ) .await; } - AuditTickResult::Failed { evidence } => { + AuditTickResult::Failed { + evidence, + no_response_class, + } => { if let FailureEvidence::AuditFailure { challenged_peer, confirmed_failed_keys, @@ -3840,8 +4359,10 @@ async fn handle_subtree_audit_result( // Rich diagnostics (from main's audit-failure logging) + the // first-failed-key correlation handle. let first_failed_key = first_failed_key_label(confirmed_failed_keys); + let audit_failure_class = no_response_class.unwrap_or("confirmed"); error!( - "Audit failure for {challenged_peer}: reason={reason:?}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + "Audit failure for {challenged_peer}: reason={reason:?}, audit_failure_class={}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + audit_failure_class, confirmed_failed_keys.len(), summary.challenged_keys, summary.absent_keys, @@ -3947,7 +4468,10 @@ async fn handle_audit_result( ) .await; } - AuditTickResult::Failed { evidence } => { + AuditTickResult::Failed { + evidence, + no_response_class, + } => { if let FailureEvidence::AuditFailure { challenged_peer, confirmed_failed_keys, @@ -3957,8 +4481,10 @@ async fn handle_audit_result( } = evidence { let first_failed_key = first_failed_key_label(confirmed_failed_keys); + let audit_failure_class = no_response_class.unwrap_or("confirmed"); error!( - "Audit failure for {challenged_peer}: reason={reason:?}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + "Audit failure for {challenged_peer}: reason={reason:?}, audit_failure_class={}, confirmed_failed_keys={}, challenged_keys={}, absent_keys={}, digest_mismatch_keys={}, first_failed_key={first_failed_key}", + audit_failure_class, confirmed_failed_keys.len(), summary.challenged_keys, summary.absent_keys, @@ -4624,17 +5150,23 @@ async fn rebuild_and_rotate_commitment( #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::{ + add_replica_hint_sender_source, admit_audit_responder, apply_audit_failure_credit_revocation, audit_failure_clears_bootstrap_claim, audit_failure_revokes_holder_credit, audit_launch_decision, config, cooldown_allows_audit, - first_failed_key_label, fresh_offer_payment_context, paid_notify_payment_context, + first_failed_key_label, fresh_offer_payment_context, handle_replication_event_recv_error, + paid_notify_payment_context, AuditResponderClass, }; use crate::payment::VerificationContext; + use crate::replication::audit_metrics; use crate::replication::recent_provers::RecentProvers; - use crate::replication::types::AuditFailureReason; + use crate::replication::types::{AuditFailureReason, HintPipeline}; + use crate::replication::{audit, possession, pruning}; use saorsa_core::identity::PeerId; use std::collections::HashMap; + use std::sync::Arc; use std::time::Duration; use std::time::Instant; + use tokio::sync::{RwLock, Semaphore}; fn test_peer(b: u8) -> PeerId { let mut bytes = [0u8; 32]; @@ -4664,6 +5196,109 @@ mod tests { ); } + #[tokio::test] + async fn replication_branch_lagged_events_are_counted() { + let before = audit_metrics::replication_event_lagged_total(); + handle_replication_event_recv_error(&tokio::sync::broadcast::error::RecvError::Lagged(3)); + let after = audit_metrics::replication_event_lagged_total(); + assert_eq!(after.saturating_sub(before), 3); + } + + #[tokio::test] + async fn digest_admission_gets_higher_per_peer_cap_subtree_stays_at_two() { + let peer = test_peer(0x44); + let semaphore = Arc::new(Semaphore::new(config::MAX_CONCURRENT_AUDIT_RESPONSES)); + + let digest_inflight = Arc::new(RwLock::new(HashMap::new())); + let mut digest_guards = Vec::new(); + for _ in 0..config::MAX_DIGEST_AUDIT_RESPONSES_PER_PEER { + let guard = admit_audit_responder( + &semaphore, + &digest_inflight, + &peer, + AuditResponderClass::Digest, + ) + .await; + assert!(guard.is_some()); + digest_guards.push(guard); + } + assert!( + admit_audit_responder( + &semaphore, + &digest_inflight, + &peer, + AuditResponderClass::Digest, + ) + .await + .is_none(), + "digest class must stop at its documented per-source cap" + ); + drop(digest_guards); + + let subtree_inflight = Arc::new(RwLock::new(HashMap::new())); + let mut subtree_guards = Vec::new(); + for _ in 0..config::MAX_AUDIT_RESPONSES_PER_PEER { + let guard = admit_audit_responder( + &semaphore, + &subtree_inflight, + &peer, + AuditResponderClass::Subtree, + ) + .await; + assert!(guard.is_some()); + subtree_guards.push(guard); + } + assert!( + admit_audit_responder( + &semaphore, + &subtree_inflight, + &peer, + AuditResponderClass::Subtree, + ) + .await + .is_none(), + "subtree class must retain the deployed cap of two" + ); + drop(subtree_guards); + } + + #[test] + fn in_scope_audit_deadlines_share_one_formula() { + let config = config::ReplicationConfig::default(); + for key_count in [1, 4, 16] { + assert_eq!( + audit::responsible_audit_response_timeout(&config, key_count), + config.audit_response_timeout(key_count) + ); + assert_eq!( + pruning::prune_audit_response_timeout(&config, key_count), + config.audit_response_timeout(key_count) + ); + } + assert_eq!( + possession::possession_probe_response_timeout(&config), + config.audit_response_timeout(1) + ); + } + + #[test] + fn replica_hint_sender_is_added_as_fallback_fetch_source() { + const EXISTING_SOURCE_ID: u8 = 1; + const HINT_SENDER_ID: u8 = 2; + const PAID_ONLY_SENDER_ID: u8 = 3; + + let existing_source = test_peer(EXISTING_SOURCE_ID); + let hint_sender = test_peer(HINT_SENDER_ID); + let paid_only_sender = test_peer(PAID_ONLY_SENDER_ID); + let mut sources = vec![existing_source]; + + add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + add_replica_hint_sender_source(&mut sources, HintPipeline::Replica, hint_sender); + add_replica_hint_sender_source(&mut sources, HintPipeline::PaidOnly, paid_only_sender); + + assert_eq!(sources, vec![existing_source, hint_sender]); + } + #[test] fn audit_timeout_preserves_active_bootstrap_claim() { assert!(!audit_failure_clears_bootstrap_claim( diff --git a/src/replication/possession.rs b/src/replication/possession.rs index 71869218..2d065185 100644 --- a/src/replication/possession.rs +++ b/src/replication/possession.rs @@ -31,6 +31,8 @@ use tokio_util::sync::CancellationToken; use crate::ant_protocol::XorName; use crate::logging::{debug, warn}; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::{self, AuditFailureClass, AuditType}; use crate::replication::config::{ ReplicationConfig, AUDIT_FAILURE_TRUST_WEIGHT, REPLICATION_PROTOCOL_ID, }; @@ -47,6 +49,10 @@ use super::REPLICATION_TRUST_WEIGHT; /// budget is the audit-response timeout sized for a single chunk. const POSSESSION_PROBE_KEY_COUNT: usize = 1; +pub(crate) fn possession_probe_response_timeout(config: &ReplicationConfig) -> Duration { + config.audit_response_timeout(POSSESSION_PROBE_KEY_COUNT) +} + /// A scheduled possession check for one freshly-replicated chunk. pub struct PossessionCheckEvent { /// Content-address of the chunk. @@ -63,9 +69,9 @@ enum ProbeOutcome { /// Peer failed the audit challenge: absent sentinel, digest mismatch, /// rejection, mismatched challenge ID, wrong digest count, or malformed reply. Failed, - /// No response (transport error / deadline). Penalised immediately at - /// audit-failure severity. - Timeout, + /// No response. Penalised immediately at audit-failure severity; the class + /// is node-local observability only. + NoResponse(AuditFailureClass), /// Peer returned a matching bootstrap claim. Graced only through the shared /// bootstrap-claim tracker. BootstrapClaim, @@ -97,6 +103,7 @@ pub fn random_delay(min: Duration, max: Duration) -> Duration { /// /// A peer that fails to prove possession, including by timeout, is penalised at /// `AuditChallenge` severity immediately. A responsive peer is left unrewarded. +#[allow(clippy::too_many_arguments)] pub(crate) async fn run_possession_check( key: XorName, peers: Vec, @@ -104,6 +111,7 @@ pub(crate) async fn run_possession_check( storage: &Arc, config: &ReplicationConfig, sync_state: &Arc>, + audit_challenge_coordinator: &Arc, shutdown: &CancellationToken, ) { let key_hex = hex::encode(key); @@ -127,29 +135,33 @@ pub(crate) async fn run_possession_check( // Single-key probe budget, matched to the audit response timeout's // bandwidth-calibrated deadline (tight enough that a relay that must refetch // the bytes blows it, generous for an honest local-disk read). - let probe_timeout = config.audit_response_timeout(POSSESSION_PROBE_KEY_COUNT); + let probe_timeout = possession_probe_response_timeout(config); for peer in peers { if shutdown.is_cancelled() { return; } - match probe_once(&key, &local_bytes, &peer, p2p_node, probe_timeout).await { + match probe_once( + &key, + &local_bytes, + &peer, + p2p_node, + audit_challenge_coordinator, + probe_timeout, + ) + .await + { ProbeOutcome::Present => { debug!("Possession check: {peer} proved possession of {key_hex}"); clear_possession_bootstrap_claim(&peer, sync_state).await; } ProbeOutcome::Failed => { clear_possession_bootstrap_claim(&peer, sync_state).await; - report_possession_audit_failure( - &peer, - &key_hex, - "failed to prove possession", - p2p_node, - ) - .await; + report_possession_confirmed_failure(&peer, &key_hex, p2p_node).await; } - ProbeOutcome::Timeout => { - report_possession_audit_failure(&peer, &key_hex, "timed out", p2p_node).await; + ProbeOutcome::NoResponse(class) => { + audit_metrics::record_audit_no_response(AuditType::Possession, class); + report_possession_audit_failure(&peer, &key_hex, class, p2p_node).await; } ProbeOutcome::BootstrapClaim => { handle_possession_bootstrap_claim(&peer, &key_hex, p2p_node, config, sync_state) @@ -171,13 +183,42 @@ async fn clear_possession_bootstrap_claim( sync_state.write().await.clear_active_bootstrap_claim(peer); } +async fn report_possession_confirmed_failure( + peer: &PeerId, + key_hex: &str, + p2p_node: &Arc, +) { + warn!( + audit_type = AuditType::Possession.as_str(), + audit_failure_class = "confirmed", + peer = %peer, + key = %key_hex, + trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, + "Possession check: {peer} failed to prove possession for {key_hex}; penalising at audit severity" + ); + p2p_node + .report_trust_event( + peer, + TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), + ) + .await; +} + async fn report_possession_audit_failure( peer: &PeerId, key_hex: &str, - reason: &str, + failure_class: AuditFailureClass, p2p_node: &Arc, ) { - warn!("Possession check: {peer} {reason} for {key_hex}; penalising at audit severity"); + warn!( + audit_type = AuditType::Possession.as_str(), + audit_failure_class = failure_class.as_str(), + peer = %peer, + key = %key_hex, + trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, + "Possession check: {peer} {} for {key_hex}; penalising at audit severity", + failure_class.as_str() + ); p2p_node .report_trust_event( peer, @@ -252,6 +293,7 @@ async fn probe_once( local_bytes: &[u8], peer: &PeerId, p2p_node: &Arc, + audit_challenge_coordinator: &Arc, probe_timeout: Duration, ) -> ProbeOutcome { // Fresh nonce per probe so a stored digest cannot be replayed, and bind the @@ -280,14 +322,26 @@ async fn probe_once( return ProbeOutcome::Inconclusive; }; + let Some(_slot) = audit_challenge_coordinator.acquire(*peer).await else { + warn!("Failed to acquire possession audit coordinator slot for {peer}"); + return ProbeOutcome::Inconclusive; + }; let response = match p2p_node .send_request(peer, REPLICATION_PROTOCOL_ID, encoded, probe_timeout) .await { Ok(response) => response, Err(e) => { - debug!("Possession probe to {peer} got no response: {e}"); - return ProbeOutcome::Timeout; + let error = e.to_string(); + let (send_error_class, audit_failure_class) = + audit_metrics::classify_audit_send_error(&error); + debug!( + audit_type = AuditType::Possession.as_str(), + audit_failure_class = audit_failure_class.as_str(), + send_error_class, + "Possession probe to {peer} got no response: {e}" + ); + return ProbeOutcome::NoResponse(audit_failure_class); } }; diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 79db6c9a..3aa73594 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -17,6 +17,8 @@ use saorsa_core::{DHTNode, P2PNode}; use tokio::sync::RwLock; use crate::ant_protocol::XorName; +use crate::replication::audit_coordinator::AuditChallengeCoordinator; +use crate::replication::audit_metrics::{self, AuditFailureClass, AuditType}; use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ storage_admission_width, ReplicationConfig, AUDIT_FAILURE_TRUST_WEIGHT, @@ -94,6 +96,8 @@ pub struct PrunePassContext<'a> { /// round-2 byte challenge cannot false-positive an honest node). `None` on /// the legacy/test-only prune path, which keeps the pre-retention behavior. pub commitment_state: Option<&'a Arc>, + /// Shared outbound limiter for digest `AuditChallenge`s. + pub audit_challenge_coordinator: &'a Arc, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -103,6 +107,12 @@ enum PruneAuditStatus { Bootstrapping, } +enum PruneAuditChallengeResult { + Response(Box), + NoResponse(AuditFailureClass), + MalformedResponse, +} + #[derive(Debug, Default)] struct RecordPruneStats { marked: usize, @@ -236,6 +246,7 @@ pub async fn run_prune_pass( allow_remote_prune_audits: bool, ) -> PruneResult { let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); run_prune_pass_with_context(PrunePassContext { self_id, storage, @@ -249,6 +260,7 @@ pub async fn run_prune_pass( repair_proof_now: None, allow_remote_prune_audits, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await } @@ -377,6 +389,7 @@ async fn prune_stored_records(ctx: &PrunePassContext<'_>) -> (usize, RecordPrune ctx.p2p_node, ctx.config, ctx.sync_state, + ctx.audit_challenge_coordinator, ) .await; let (keys_to_delete, revalidated_cleared) = revalidated_record_prune_keys( @@ -951,6 +964,7 @@ async fn collect_record_prune_proofs( p2p_node: &Arc, config: &ReplicationConfig, sync_state: &Arc>, + audit_challenge_coordinator: &Arc, ) -> HashMap> { if candidates.is_empty() { return HashMap::new(); @@ -966,6 +980,7 @@ async fn collect_record_prune_proofs( p2p_node, config, sync_state, + audit_challenge_coordinator, &report_state, ) }) @@ -1136,6 +1151,7 @@ fn target_peers_reported_present( /// Challenge a peer to prove it holds the exact record bytes for `key`. /// `None` means the peer failed to provide usable proof. +#[allow(clippy::too_many_arguments)] async fn peer_proves_record( peer: PeerId, key: XorName, @@ -1143,6 +1159,7 @@ async fn peer_proves_record( p2p_node: &Arc, config: &ReplicationConfig, sync_state: &Arc>, + audit_challenge_coordinator: &Arc, report_state: &PruneAuditReportState, ) -> Option<(PeerId, XorName)> { let local_bytes = local_record_bytes(&key, storage).await?; @@ -1152,14 +1169,39 @@ async fn peer_proves_record( (rng.gen::(), rng.gen::<[u8; 32]>()) }; let (encoded, key_count) = encode_prune_audit_challenge(&peer, key, challenge_id, nonce)?; - let Some(decoded) = - send_prune_audit_challenge(&peer, &key, encoded, key_count, p2p_node, config).await - else { - // No decoded response means a timeout or malformed reply. Prune - // confirmation reuses `AuditChallenge` semantics, so this is an immediate - // audit failure just like a decoded bad proof below. - report_prune_audit_failure_once(&peer, &key, p2p_node, config, report_state).await; - return None; + let decoded = match send_prune_audit_challenge( + &peer, + &key, + encoded, + key_count, + p2p_node, + config, + audit_challenge_coordinator, + ) + .await + { + PruneAuditChallengeResult::Response(decoded) => *decoded, + PruneAuditChallengeResult::NoResponse(class) => { + // No response means an immediate audit failure, but keep the local + // class split so timeout metrics are not polluted by pre-delivery + // failures. + audit_metrics::record_audit_no_response(AuditType::Prune, class); + report_prune_audit_failure_once( + &peer, + &key, + p2p_node, + config, + report_state, + Some(class), + ) + .await; + return None; + } + PruneAuditChallengeResult::MalformedResponse => { + report_prune_audit_failure_once(&peer, &key, p2p_node, config, report_state, None) + .await; + return None; + } }; let status = @@ -1176,7 +1218,8 @@ async fn peer_proves_record( None } PruneAuditStatus::Failed => { - report_prune_audit_failure_once(&peer, &key, p2p_node, config, report_state).await; + report_prune_audit_failure_once(&peer, &key, p2p_node, config, report_state, None) + .await; None } } @@ -1221,6 +1264,13 @@ fn encode_prune_audit_challenge( Some((encoded, key_count)) } +pub(crate) fn prune_audit_response_timeout( + config: &ReplicationConfig, + key_count: usize, +) -> Duration { + config.audit_response_timeout(key_count) +} + async fn send_prune_audit_challenge( peer: &PeerId, key: &XorName, @@ -1228,19 +1278,33 @@ async fn send_prune_audit_challenge( key_count: usize, p2p_node: &Arc, config: &ReplicationConfig, -) -> Option { - let timeout = config.audit_response_timeout(key_count); + audit_challenge_coordinator: &Arc, +) -> PruneAuditChallengeResult { + let Some(_slot) = audit_challenge_coordinator.acquire(*peer).await else { + warn!( + "Prune audit challenge for {} against {peer} could not acquire coordinator slot", + hex::encode(key) + ); + return PruneAuditChallengeResult::MalformedResponse; + }; + let timeout = prune_audit_response_timeout(config, key_count); let response = match p2p_node .send_request(peer, REPLICATION_PROTOCOL_ID, encoded, timeout) .await { Ok(response) => response, Err(e) => { + let error = e.to_string(); + let (send_error_class, audit_failure_class) = + audit_metrics::classify_audit_send_error(&error); debug!( + audit_type = AuditType::Prune.as_str(), + audit_failure_class = audit_failure_class.as_str(), + send_error_class, "Prune audit challenge for {} against {peer} failed: {e}", hex::encode(key) ); - return None; + return PruneAuditChallengeResult::NoResponse(audit_failure_class); } }; @@ -1248,11 +1312,11 @@ async fn send_prune_audit_challenge( Ok(msg) => msg, Err(e) => { warn!("Failed to decode prune audit response from {peer}: {e}"); - return None; + return PruneAuditChallengeResult::MalformedResponse; } }; - Some(decoded) + PruneAuditChallengeResult::Response(Box::new(decoded)) } fn prune_audit_response_status( @@ -1371,6 +1435,7 @@ async fn report_prune_audit_failure_once( p2p_node: &Arc, config: &ReplicationConfig, report_state: &PruneAuditReportState, + failure_class: Option, ) -> bool { let should_report = peer_is_currently_responsible(peer, key, p2p_node, config).await && reserve_prune_audit_failure_report(report_state, peer).await; @@ -1378,6 +1443,16 @@ async fn report_prune_audit_failure_once( return false; } + let audit_failure_class = failure_class.map_or("failed", AuditFailureClass::as_str); + warn!( + audit_type = AuditType::Prune.as_str(), + audit_failure_class, + peer = %peer, + key = %hex::encode(key), + trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, + "Prune audit failure: peer={peer}, audit_failure_class={audit_failure_class}, key={}", + hex::encode(key) + ); p2p_node .report_trust_event( peer, diff --git a/src/replication/quorum.rs b/src/replication/quorum.rs index 78211a08..5a5fbbec 100644 --- a/src/replication/quorum.rs +++ b/src/replication/quorum.rs @@ -5,14 +5,18 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use crate::logging::{debug, info, warn}; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; +use tokio::task::JoinHandle; use crate::ant_protocol::XorName; -use crate::replication::config::{ReplicationConfig, REPLICATION_PROTOCOL_ID}; +use crate::replication::config::{ + ReplicationConfig, MAX_VERIFICATION_KEYS_PER_REQUEST, PAID_LIST_CLOSE_GROUP_SIZE, + PAID_LIST_FLEX_EDGE_COUNT, REPLICATION_PROTOCOL_ID, +}; use crate::replication::protocol::{ ReplicationMessage, ReplicationMessageBody, VerificationRequest, VerificationResponse, }; @@ -21,6 +25,19 @@ use crate::replication::types::{KeyVerificationEvidence, PaidListEvidence, Prese /// Verification round duration that is worth surfacing at info level. const VERIFICATION_ROUND_SLOW_LOG_MS: u128 = 500; +struct VerificationBatchResult { + peer: PeerId, + requested_keys: Vec, + response: Option, +} + +struct PaidListVoteSummary { + confirmed: usize, + effective_group_size: usize, + max_possible_confirmed: usize, + max_possible_group_size: usize, +} + // --------------------------------------------------------------------------- // Verification targets // --------------------------------------------------------------------------- @@ -36,6 +53,14 @@ pub struct VerificationTargets { /// Per-key: self-inclusive paid close-group size used to compute /// `ConfirmNeeded(K)`. pub paid_group_sizes: HashMap, + /// Per-key: remote peers in the furthest paid-list positions. + /// + /// These peers are queried, but only positive paid-list evidence from them + /// expands the paid-list majority denominator once the paid group reaches + /// the configured 20-peer width. Negative/missing edge evidence is ignored + /// for that full-width paid-list quorum because boundary peers can + /// legitimately differ under churn. + pub paid_edge_targets: HashMap>, /// Union of all target peers across all keys. pub all_peers: HashSet, /// Which keys each peer should be queried about. @@ -60,6 +85,7 @@ pub async fn compute_verification_targets( quorum_targets: HashMap::new(), paid_targets: HashMap::new(), paid_group_sizes: HashMap::new(), + paid_edge_targets: HashMap::new(), all_peers: HashSet::new(), peer_to_keys: HashMap::new(), peer_to_paid_keys: HashMap::new(), @@ -82,11 +108,18 @@ pub async fn compute_verification_targets( .find_closest_nodes_local_with_self(&key, config.paid_list_close_group_size) .await; let paid_group_size = paid_closest.len(); - let paid_peers: Vec = paid_closest - .iter() - .filter(|n| n.peer_id != *self_id) - .map(|n| n.peer_id) - .collect(); + let paid_edge_start = paid_group_size.saturating_sub(PAID_LIST_FLEX_EDGE_COUNT); + let mut paid_peers = Vec::new(); + let mut paid_edge_peers = HashSet::new(); + for (idx, node) in paid_closest.iter().enumerate() { + if node.peer_id == *self_id { + continue; + } + paid_peers.push(node.peer_id); + if idx >= paid_edge_start { + paid_edge_peers.insert(node.peer_id); + } + } // VerifyTargets = PaidTargets ∪ QuorumTargets for &peer in &quorum_peers { @@ -106,6 +139,7 @@ pub async fn compute_verification_targets( targets.quorum_targets.insert(key, quorum_peers); targets.paid_targets.insert(key, paid_peers); targets.paid_group_sizes.insert(key, paid_group_size); + targets.paid_edge_targets.insert(key, paid_edge_peers); } // Deduplicate keys per peer (a peer in both quorum and paid targets for @@ -133,6 +167,7 @@ pub async fn compute_presence_targets( quorum_targets: HashMap::new(), paid_targets: HashMap::new(), paid_group_sizes: HashMap::new(), + paid_edge_targets: HashMap::new(), all_peers: HashSet::new(), peer_to_keys: HashMap::new(), peer_to_paid_keys: HashMap::new(), @@ -263,25 +298,9 @@ pub fn evaluate_key_evidence_with_holder_check( let paid_peers = targets.paid_targets.get(key).map_or(&[][..], Vec::as_slice); let present_peers = collect_present_sources(evidence, quorum_peers, paid_peers); - // Count paid-list evidence from PaidTargets. - let mut paid_confirmed = 0usize; - let mut paid_unresolved = 0usize; - - for peer in paid_peers { - match evidence.paid_list.get(peer) { - Some(PaidListEvidence::Confirmed) => paid_confirmed += 1, - Some(PaidListEvidence::NotFound) => {} - Some(PaidListEvidence::Unresolved) | None => paid_unresolved += 1, - } - } - let quorum_needed = config.quorum_needed(quorum_peers.len()); - let paid_group_size = targets - .paid_group_sizes - .get(key) - .copied() - .unwrap_or(paid_peers.len()); - let confirm_needed = ReplicationConfig::confirm_needed(paid_group_size); + let paid_votes = summarize_paid_list_votes(key, evidence, targets, paid_peers); + let confirm_needed = ReplicationConfig::confirm_needed(paid_votes.effective_group_size); // Step 10: Presence quorum reached. // quorum_needed == 0 means zero targets exist — quorum is impossible, @@ -295,14 +314,16 @@ pub fn evaluate_key_evidence_with_holder_check( // Step 9: Paid-list majority reached. // confirm_needed from 0 paid peers is 1, so this naturally fails with // 0 confirmed — no special guard needed. But be explicit for clarity. - if paid_group_size > 0 && paid_confirmed >= confirm_needed { + if paid_votes.effective_group_size > 0 && paid_votes.confirmed >= confirm_needed { return KeyVerificationOutcome::PaidListVerified { sources: present_peers, }; } // Step 14: Fail fast when both paths are impossible. - let paid_possible = paid_group_size > 0 && paid_confirmed + paid_unresolved >= confirm_needed; + let max_confirm_needed = ReplicationConfig::confirm_needed(paid_votes.max_possible_group_size); + let paid_possible = paid_votes.max_possible_group_size > 0 + && paid_votes.max_possible_confirmed >= max_confirm_needed; let quorum_possible = quorum_needed > 0 && presence_positive + presence_unresolved >= quorum_needed; @@ -314,6 +335,61 @@ pub fn evaluate_key_evidence_with_holder_check( KeyVerificationOutcome::QuorumInconclusive } +fn summarize_paid_list_votes( + key: &XorName, + evidence: &KeyVerificationEvidence, + targets: &VerificationTargets, + paid_peers: &[PeerId], +) -> PaidListVoteSummary { + let paid_group_size = targets + .paid_group_sizes + .get(key) + .copied() + .unwrap_or(paid_peers.len()); + let paid_edge_count = if paid_group_size >= PAID_LIST_CLOSE_GROUP_SIZE { + PAID_LIST_FLEX_EDGE_COUNT.min(paid_group_size) + } else { + 0 + }; + let core_group_size = paid_group_size.saturating_sub(paid_edge_count); + let edge_targets = (paid_edge_count > 0) + .then(|| targets.paid_edge_targets.get(key)) + .flatten(); + + let mut confirmed = 0usize; + let mut confirmed_edge = 0usize; + let mut unresolved_core = 0usize; + let mut unresolved_edge = 0usize; + + for peer in paid_peers { + let is_edge = edge_targets.is_some_and(|peers| peers.contains(peer)); + match evidence.paid_list.get(peer) { + Some(PaidListEvidence::Confirmed) => { + confirmed += 1; + if is_edge { + confirmed_edge += 1; + } + } + Some(PaidListEvidence::NotFound) => {} + Some(PaidListEvidence::Unresolved) | None => { + if is_edge { + unresolved_edge += 1; + } else { + unresolved_core += 1; + } + } + } + } + + let effective_group_size = core_group_size + confirmed_edge; + PaidListVoteSummary { + confirmed, + effective_group_size, + max_possible_confirmed: confirmed + unresolved_core + unresolved_edge, + max_possible_group_size: effective_group_size + unresolved_edge, + } +} + /// Return peers that gave positive presence evidence for a key. /// /// Only peers in the computed verification target sets are considered. @@ -351,6 +427,39 @@ fn collect_present_sources( present_peers } +fn verification_requests_for_peer( + peer_keys: &[XorName], + paid_check_keys: Option<&HashSet>, +) -> Vec { + peer_keys + .chunks(MAX_VERIFICATION_KEYS_PER_REQUEST) + .map(|key_batch| VerificationRequest { + keys: key_batch.to_vec(), + paid_list_check_indices: paid_indices_for_key_batch(key_batch, paid_check_keys), + }) + .collect() +} + +fn paid_indices_for_key_batch( + key_batch: &[XorName], + paid_check_keys: Option<&HashSet>, +) -> Vec { + let Some(paid_keys) = paid_check_keys else { + return Vec::new(); + }; + + key_batch + .iter() + .enumerate() + .filter_map(|(idx, key)| { + paid_keys + .contains(key) + .then_some(idx) + .and_then(|idx| u32::try_from(idx).ok()) + }) + .collect() +} + // --------------------------------------------------------------------------- // Network verification round // --------------------------------------------------------------------------- @@ -383,126 +492,171 @@ pub async fn run_verification_round( }) .collect(); - // Send one batched request per peer. + let handles = + spawn_verification_batch_tasks(targets, p2p_node, config.verification_request_timeout); + collect_verification_batch_results(handles, targets, &mut evidence).await; + + let elapsed_ms = started.elapsed().as_millis(); + let batch_count = targets + .peer_to_keys + .values() + .map(|peer_keys| peer_keys.chunks(MAX_VERIFICATION_KEYS_PER_REQUEST).count()) + .sum::(); + if elapsed_ms >= VERIFICATION_ROUND_SLOW_LOG_MS { + info!( + target: "ant_node::replication::verification", + "Slow quorum verification round: keys={}, peers={peer_count}, batches={batch_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", + keys.len(), + ); + } else { + debug!( + target: "ant_node::replication::verification", + "Quorum verification round: keys={}, peers={peer_count}, batches={batch_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", + keys.len(), + ); + } + + evidence +} + +fn spawn_verification_batch_tasks( + targets: &VerificationTargets, + p2p_node: &Arc, + timeout: Duration, +) -> Vec> { let mut handles = Vec::new(); for (&peer, peer_keys) in &targets.peer_to_keys { let paid_check_keys = targets.peer_to_paid_keys.get(&peer); - // Build paid_list_check_indices: which of this peer's keys need - // paid-list status. - let mut paid_indices = Vec::new(); - for (i, key) in peer_keys.iter().enumerate() { - if let Some(paid_keys) = paid_check_keys { - if paid_keys.contains(key) { - if let Ok(idx) = u32::try_from(i) { - paid_indices.push(idx); - } - } - } + for request in verification_requests_for_peer(peer_keys, paid_check_keys) { + let requested_keys = request.keys.clone(); + let msg = ReplicationMessage { + request_id: rand::random(), + body: ReplicationMessageBody::VerificationRequest(request), + }; + + handles.push(spawn_verification_batch_task( + peer, + requested_keys, + msg, + Arc::clone(p2p_node), + timeout, + )); } + } - let request = VerificationRequest { - keys: peer_keys.clone(), - paid_list_check_indices: paid_indices, - }; + handles +} - let msg = ReplicationMessage { - request_id: rand::random(), - body: ReplicationMessageBody::VerificationRequest(request), +fn spawn_verification_batch_task( + peer: PeerId, + requested_keys: Vec, + msg: ReplicationMessage, + p2p: Arc, + timeout: Duration, +) -> JoinHandle { + tokio::spawn(async move { + let encoded = match msg.encode() { + Ok(data) => data, + Err(e) => { + warn!("Failed to encode verification request: {e}"); + return VerificationBatchResult { + peer, + requested_keys, + response: None, + }; + } }; - let p2p = Arc::clone(p2p_node); - let timeout = config.verification_request_timeout; - let peer_id = peer; - - handles.push(tokio::spawn(async move { - let encoded = match msg.encode() { - Ok(data) => data, - Err(e) => { - warn!("Failed to encode verification request: {e}"); - return (peer_id, None); - } - }; - - match p2p - .send_request(&peer_id, REPLICATION_PROTOCOL_ID, encoded, timeout) - .await - { - Ok(response) => match ReplicationMessage::decode(&response.data) { - Ok(decoded) => (peer_id, Some(decoded)), - Err(e) => { - warn!("Failed to decode verification response from {peer_id}: {e}"); - (peer_id, None) - } - }, + let response = match p2p + .send_request(&peer, REPLICATION_PROTOCOL_ID, encoded, timeout) + .await + { + Ok(response) => match ReplicationMessage::decode(&response.data) { + Ok(decoded) => Some(decoded), Err(e) => { - debug!("Verification request to {peer_id} failed: {e}"); - (peer_id, None) + warn!("Failed to decode verification response from {peer}: {e}"); + None } + }, + Err(e) => { + debug!("Verification request to {peer} failed: {e}"); + None } - })); - } + }; + + VerificationBatchResult { + peer, + requested_keys, + response, + } + }) +} - // Collect responses. +async fn collect_verification_batch_results( + handles: Vec>, + targets: &VerificationTargets, + evidence: &mut HashMap, +) { for handle in handles { - let (peer, response) = match handle.await { + let batch = match handle.await { Ok(result) => result, Err(e) => { warn!("Verification task panicked: {e}"); continue; } }; + let peer = batch.peer; - let Some(msg) = response else { - // Timeout/error: mark all keys for this peer as unresolved. - mark_peer_unresolved(&peer, targets, &mut evidence); + let Some(msg) = batch.response else { + mark_peer_keys_unresolved(&peer, &batch.requested_keys, targets, evidence); continue; }; if let ReplicationMessageBody::VerificationResponse(resp) = msg.body { - process_verification_response(&peer, &resp, targets, &mut evidence); + process_verification_response_for_keys( + &peer, + &batch.requested_keys, + &resp, + targets, + evidence, + ); } } - - let elapsed_ms = started.elapsed().as_millis(); - if elapsed_ms >= VERIFICATION_ROUND_SLOW_LOG_MS { - info!( - target: "ant_node::replication::verification", - "Slow quorum verification round: keys={}, peers={peer_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", - keys.len(), - ); - } else { - debug!( - target: "ant_node::replication::verification", - "Quorum verification round: keys={}, peers={peer_count}, requested_key_refs={requested_key_refs}, elapsed_ms={elapsed_ms}", - keys.len(), - ); - } - - evidence } /// Mark all keys for a peer as unresolved (timeout / decode failure). +#[cfg(test)] fn mark_peer_unresolved( peer: &PeerId, targets: &VerificationTargets, evidence: &mut HashMap, ) { if let Some(peer_keys) = targets.peer_to_keys.get(peer) { - let is_paid_peer = targets.peer_to_paid_keys.get(peer); - for key in peer_keys { - if let Some(ev) = evidence.get_mut(key) { - ev.presence.insert(*peer, PresenceEvidence::Unresolved); - if is_paid_peer.is_some_and(|ks| ks.contains(key)) { - ev.paid_list.insert(*peer, PaidListEvidence::Unresolved); - } + mark_peer_keys_unresolved(peer, peer_keys, targets, evidence); + } +} + +fn mark_peer_keys_unresolved( + peer: &PeerId, + requested_keys: &[XorName], + targets: &VerificationTargets, + evidence: &mut HashMap, +) { + let paid_check_keys = targets.peer_to_paid_keys.get(peer); + for key in requested_keys { + if let Some(ev) = evidence.get_mut(key) { + ev.presence.insert(*peer, PresenceEvidence::Unresolved); + if paid_check_keys.is_some_and(|ks| ks.contains(key)) { + ev.paid_list.insert(*peer, PaidListEvidence::Unresolved); } } } } /// Process a single peer's verification response into the evidence map. +#[cfg(test)] fn process_verification_response( peer: &PeerId, response: &VerificationResponse, @@ -513,18 +667,30 @@ fn process_verification_response( return; }; + process_verification_response_for_keys(peer, peer_keys, response, targets, evidence); +} + +fn process_verification_response_for_keys( + peer: &PeerId, + requested_keys: &[XorName], + response: &VerificationResponse, + targets: &VerificationTargets, + evidence: &mut HashMap, +) { + let paid_check_keys = targets.peer_to_paid_keys.get(peer); + // Use a HashSet for O(1) key membership checks instead of linear scan, // preventing CPU amplification from large responses. - let peer_keys_set: HashSet<&XorName> = peer_keys.iter().collect(); + let requested_keys_set: HashSet<&XorName> = requested_keys.iter().collect(); // Cap results at 2x requested keys to limit processing of stuffed // responses while still tolerating some unsolicited entries. - let max_results = peer_keys.len().saturating_mul(2); + let max_results = requested_keys.len().saturating_mul(2); let results = if response.results.len() > max_results { warn!( "Peer {peer} sent {} verification results but only {} keys were requested — truncating", response.results.len(), - peer_keys.len(), + requested_keys.len(), ); &response.results[..max_results] } else { @@ -533,7 +699,7 @@ fn process_verification_response( // Match response results to requested keys. for result in results { - if !peer_keys_set.contains(&result.key) { + if !requested_keys_set.contains(&result.key) { continue; // Ignore unsolicited key results. } @@ -547,25 +713,26 @@ fn process_verification_response( ev.presence.insert(*peer, presence); // Paid-list evidence (only if requested). - if let Some(is_paid) = result.paid { - let paid = if is_paid { - PaidListEvidence::Confirmed - } else { - PaidListEvidence::NotFound - }; - ev.paid_list.insert(*peer, paid); + if paid_check_keys.is_some_and(|ks| ks.contains(&result.key)) { + if let Some(is_paid) = result.paid { + let paid = if is_paid { + PaidListEvidence::Confirmed + } else { + PaidListEvidence::NotFound + }; + ev.paid_list.insert(*peer, paid); + } } } } // Keys that were requested but not in response -> unresolved. - let is_paid_peer = targets.peer_to_paid_keys.get(peer); - for key in peer_keys { + for key in requested_keys { if let Some(ev) = evidence.get_mut(key) { ev.presence .entry(*peer) .or_insert(PresenceEvidence::Unresolved); - if is_paid_peer.is_some_and(|ks| ks.contains(key)) { + if paid_check_keys.is_some_and(|ks| ks.contains(key)) { ev.paid_list .entry(*peer) .or_insert(PaidListEvidence::Unresolved); @@ -582,8 +749,26 @@ fn process_verification_response( #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; + use crate::replication::config::PAID_LIST_CLOSE_GROUP_SIZE; use crate::replication::protocol::KeyVerificationResult; + const PAID_LIST_INNER_GROUP_SIZE: usize = + PAID_LIST_CLOSE_GROUP_SIZE - PAID_LIST_FLEX_EDGE_COUNT; + const PAID_LIST_INNER_MAJORITY: usize = PAID_LIST_INNER_GROUP_SIZE / 2 + 1; + const PAID_LIST_ONE_EDGE_GROUP_SIZE: usize = PAID_LIST_INNER_GROUP_SIZE + 1; + const PAID_LIST_ONE_EDGE_MAJORITY: usize = PAID_LIST_ONE_EDGE_GROUP_SIZE / 2 + 1; + const PAID_LIST_FULL_MAJORITY: usize = PAID_LIST_CLOSE_GROUP_SIZE / 2 + 1; + const FIRST_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE; + const SECOND_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE + 1; + const THIRD_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE + 2; + const FOURTH_EDGE_INDEX: usize = PAID_LIST_INNER_GROUP_SIZE + 3; + const REMOTE_PAID_PEERS_WITH_SELF_IN_GROUP: usize = PAID_LIST_CLOSE_GROUP_SIZE - 1; + const REMOTE_EDGE_COUNT_WHEN_SELF_IN_CORE: usize = PAID_LIST_FLEX_EDGE_COUNT; + const REMOTE_EDGE_COUNT_WHEN_SELF_ON_EDGE: usize = PAID_LIST_FLEX_EDGE_COUNT - 1; + const SELF_EDGE_REMOTE_FULL_GROUP_SIZE: usize = + PAID_LIST_INNER_GROUP_SIZE + REMOTE_EDGE_COUNT_WHEN_SELF_ON_EDGE; + const SELF_EDGE_REMOTE_FULL_MAJORITY: usize = SELF_EDGE_REMOTE_FULL_GROUP_SIZE / 2 + 1; + /// Build a `PeerId` from a single byte (zero-padded to 32 bytes). fn peer_id_from_byte(b: u8) -> PeerId { let mut bytes = [0u8; 32]; @@ -591,11 +776,91 @@ mod tests { PeerId::from_bytes(bytes) } + fn peer_id_from_usize(value: usize) -> PeerId { + peer_id_from_byte(u8::try_from(value).expect("test peer id fits u8")) + } + /// Build an `XorName` from a single byte (repeated to 32 bytes). fn xor_name_from_byte(b: u8) -> XorName { [b; 32] } + fn xor_name_from_usize(value: usize) -> XorName { + let mut name = [0u8; 32]; + let bytes = u64::try_from(value) + .expect("test value fits u64") + .to_le_bytes(); + name[..bytes.len()].copy_from_slice(&bytes); + name + } + + fn paid_edge_targets_for_peers(paid_peers: &[PeerId]) -> HashSet { + paid_peers[paid_peers.len().saturating_sub(PAID_LIST_FLEX_EDGE_COUNT)..] + .iter() + .copied() + .collect() + } + + fn paid_vote_evidence( + paid_peers: &[PeerId], + confirmed_indices: &[usize], + ) -> Vec<(PeerId, PaidListEvidence)> { + let confirmed_indices: HashSet = confirmed_indices.iter().copied().collect(); + paid_peers + .iter() + .enumerate() + .map(|(idx, peer)| { + ( + *peer, + if confirmed_indices.contains(&idx) { + PaidListEvidence::Confirmed + } else { + PaidListEvidence::NotFound + }, + ) + }) + .collect() + } + + fn paid_vote_evidence_with_unresolved( + paid_peers: &[PeerId], + confirmed_indices: &[usize], + unresolved_indices: &[usize], + ) -> Vec<(PeerId, PaidListEvidence)> { + let confirmed_indices: HashSet = confirmed_indices.iter().copied().collect(); + let unresolved_indices: HashSet = unresolved_indices.iter().copied().collect(); + paid_peers + .iter() + .enumerate() + .map(|(idx, peer)| { + let status = if confirmed_indices.contains(&idx) { + PaidListEvidence::Confirmed + } else if unresolved_indices.contains(&idx) { + PaidListEvidence::Unresolved + } else { + PaidListEvidence::NotFound + }; + (*peer, status) + }) + .collect() + } + + fn self_inclusive_paid_targets( + key: &XorName, + paid_peers: &[PeerId], + remote_edge_count: usize, + ) -> VerificationTargets { + let mut targets = single_key_targets(key, vec![], paid_peers.to_vec()); + targets + .paid_group_sizes + .insert(*key, PAID_LIST_CLOSE_GROUP_SIZE); + let edge_start = paid_peers.len().saturating_sub(remote_edge_count); + targets + .paid_edge_targets + .insert(*key, paid_peers[edge_start..].iter().copied().collect()); + targets + } + /// Helper: build minimal `VerificationTargets` for a single key with /// explicit quorum and paid peer lists. fn single_key_targets( @@ -624,10 +889,12 @@ mod tests { } let paid_group_size = paid_peers.len(); + let paid_edge_targets = paid_edge_targets_for_peers(&paid_peers); VerificationTargets { quorum_targets: std::iter::once((key.to_owned(), quorum_peers)).collect(), paid_group_sizes: std::iter::once((key.to_owned(), paid_group_size)).collect(), paid_targets: std::iter::once((key.to_owned(), paid_peers)).collect(), + paid_edge_targets: std::iter::once((key.to_owned(), paid_edge_targets)).collect(), all_peers, peer_to_keys, peer_to_paid_keys, @@ -990,62 +1257,300 @@ mod tests { } #[test] - fn paid_list_majority_uses_self_inclusive_paid_group_size() { + fn paid_list_edge_notfound_votes_shrink_denominator_to_inner_group() { let key = xor_name_from_byte(0x61); let config = ReplicationConfig::default(); - // Real target computation uses PaidCloseGroup(K), which is - // self-inclusive. If self is in a 20-node paid group and does not - // already have local paid-list state, 10 remote confirmations are not - // enough: ConfirmNeeded(20) is 11. - let paid_peers: Vec = (1..=19).map(peer_id_from_byte).collect(); - let mut targets = single_key_targets(&key, vec![], paid_peers.clone()); - targets.paid_group_sizes.insert(key, 20); + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); - let ten_confirmations = build_evidence( + let below_threshold = build_evidence( vec![], - paid_peers - .iter() - .enumerate() - .map(|(i, p)| { - ( - *p, - if i < 10 { - PaidListEvidence::Confirmed - } else { - PaidListEvidence::NotFound - }, - ) - }) - .collect(), + paid_vote_evidence(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7]), ); - let outcome = evaluate_key_evidence(&key, &ten_confirmations, &targets, &config); + let outcome = evaluate_key_evidence(&key, &below_threshold, &targets, &config); assert!( matches!(outcome, KeyVerificationOutcome::QuorumFailed), - "10/20 paid confirmations must not authorize the key, got {outcome:?}" + "8/{PAID_LIST_INNER_GROUP_SIZE} paid confirmations must not authorize the key, got {outcome:?}" ); - let eleven_confirmations = build_evidence( + let threshold_confirmed = build_evidence( vec![], - paid_peers - .iter() - .enumerate() - .map(|(i, p)| { - ( - *p, - if i < 11 { - PaidListEvidence::Confirmed - } else { - PaidListEvidence::NotFound - }, - ) - }) - .collect(), + paid_vote_evidence( + &paid_peers, + &(0..PAID_LIST_INNER_MAJORITY).collect::>(), + ), + ); + let outcome = evaluate_key_evidence(&key, &threshold_confirmed, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_INNER_MAJORITY}/{PAID_LIST_INNER_GROUP_SIZE} paid confirmations should authorize the key, got {outcome:?}" + ); + } + + #[test] + fn paid_list_positive_edge_votes_expand_denominator() { + let key = xor_name_from_byte(0x62); + let config = ReplicationConfig::default(); + + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); + + let one_edge_confirmed = build_evidence( + vec![], + paid_vote_evidence(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7, THIRD_EDGE_INDEX]), + ); + let outcome = evaluate_key_evidence(&key, &one_edge_confirmed, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_ONE_EDGE_MAJORITY}/{PAID_LIST_ONE_EDGE_GROUP_SIZE} paid confirmations should authorize the key, got {outcome:?}" + ); + + let ten_with_all_edges = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + FIRST_EDGE_INDEX, + SECOND_EDGE_INDEX, + THIRD_EDGE_INDEX, + FOURTH_EDGE_INDEX, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &ten_with_all_edges, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "10/{PAID_LIST_CLOSE_GROUP_SIZE} paid confirmations must not authorize when all edge peers are positive, got {outcome:?}" + ); + + let eleven_with_all_edges = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + FIRST_EDGE_INDEX, + SECOND_EDGE_INDEX, + THIRD_EDGE_INDEX, + FOURTH_EDGE_INDEX, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &eleven_with_all_edges, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_FULL_MAJORITY}/{PAID_LIST_CLOSE_GROUP_SIZE} paid confirmations should authorize when all edge peers are positive, got {outcome:?}" + ); + } + + #[test] + fn paid_list_self_inclusive_missing_core_keeps_inner_threshold() { + let key = xor_name_from_byte(0x63); + let config = ReplicationConfig::default(); + let paid_peers: Vec = (1..=REMOTE_PAID_PEERS_WITH_SELF_IN_GROUP) + .map(peer_id_from_usize) + .collect(); + let targets = + self_inclusive_paid_targets(&key, &paid_peers, REMOTE_EDGE_COUNT_WHEN_SELF_IN_CORE); + + let below_threshold = build_evidence( + vec![], + paid_vote_evidence(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7]), + ); + let outcome = evaluate_key_evidence(&key, &below_threshold, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "8/{PAID_LIST_INNER_GROUP_SIZE} should fail when self is a missing core voter, got {outcome:?}" + ); + + let threshold_confirmed = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &(0..PAID_LIST_INNER_MAJORITY).collect::>(), + ), + ); + let outcome = evaluate_key_evidence(&key, &threshold_confirmed, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{PAID_LIST_INNER_MAJORITY}/{PAID_LIST_INNER_GROUP_SIZE} should pass when self is a missing core voter, got {outcome:?}" + ); + } + + #[test] + fn paid_list_self_inclusive_missing_edge_discounts_self_edge_only_when_negative() { + let key = xor_name_from_byte(0x64); + let config = ReplicationConfig::default(); + let paid_peers: Vec = (1..=REMOTE_PAID_PEERS_WITH_SELF_IN_GROUP) + .map(peer_id_from_usize) + .collect(); + let targets = + self_inclusive_paid_targets(&key, &paid_peers, REMOTE_EDGE_COUNT_WHEN_SELF_ON_EDGE); + + let inner_threshold_confirmed = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &(0..PAID_LIST_INNER_MAJORITY).collect::>(), + ), ); - let outcome = evaluate_key_evidence(&key, &eleven_confirmations, &targets, &config); + let outcome = evaluate_key_evidence(&key, &inner_threshold_confirmed, &targets, &config); assert!( matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), - "11/20 paid confirmations should authorize the key, got {outcome:?}" + "{PAID_LIST_INNER_MAJORITY}/{PAID_LIST_INNER_GROUP_SIZE} should pass when self is a missing edge voter, got {outcome:?}" + ); + + let all_remote_edges_below = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + PAID_LIST_INNER_GROUP_SIZE, + PAID_LIST_INNER_GROUP_SIZE + 1, + PAID_LIST_INNER_GROUP_SIZE + 2, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &all_remote_edges_below, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumFailed), + "9/{SELF_EDGE_REMOTE_FULL_GROUP_SIZE} should fail when all remote edge peers are positive but self-edge is missing, got {outcome:?}" + ); + + let all_remote_edges_threshold = build_evidence( + vec![], + paid_vote_evidence( + &paid_peers, + &[ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + PAID_LIST_INNER_GROUP_SIZE, + PAID_LIST_INNER_GROUP_SIZE + 1, + PAID_LIST_INNER_GROUP_SIZE + 2, + ], + ), + ); + let outcome = evaluate_key_evidence(&key, &all_remote_edges_threshold, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { .. }), + "{SELF_EDGE_REMOTE_FULL_MAJORITY}/{SELF_EDGE_REMOTE_FULL_GROUP_SIZE} should pass when all remote edge peers are positive but self-edge is missing, got {outcome:?}" + ); + } + + #[test] + fn paid_list_unresolved_core_or_edge_keeps_possible_round_inconclusive() { + let key = xor_name_from_byte(0x65); + let config = ReplicationConfig::default(); + let paid_peers: Vec = (1..=PAID_LIST_CLOSE_GROUP_SIZE) + .map(peer_id_from_usize) + .collect(); + let targets = single_key_targets(&key, vec![], paid_peers.clone()); + + let unresolved_core = build_evidence( + vec![], + paid_vote_evidence_with_unresolved(&paid_peers, &[0, 1, 2, 3, 4, 5, 6, 7], &[8]), + ); + let outcome = evaluate_key_evidence(&key, &unresolved_core, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumInconclusive), + "8 confirmed plus one unresolved core voter can still become 9/{PAID_LIST_INNER_GROUP_SIZE}, got {outcome:?}" + ); + + let unresolved_edge = build_evidence( + vec![], + paid_vote_evidence_with_unresolved( + &paid_peers, + &[0, 1, 2, 3, 4, 5, 6, 7], + &[FIRST_EDGE_INDEX], + ), + ); + let outcome = evaluate_key_evidence(&key, &unresolved_edge, &targets, &config); + assert!( + matches!(outcome, KeyVerificationOutcome::QuorumInconclusive), + "8 confirmed plus one unresolved edge voter can still become 9/{PAID_LIST_ONE_EDGE_GROUP_SIZE}, got {outcome:?}" + ); + } + + #[test] + fn production_paid_list_vote_authorizes_when_storage_majority_missing() { + const PRODUCTION_PAID_GROUP: u8 = 20; + const STORAGE_HOLDERS_BELOW_QUORUM: usize = 3; + const PAID_CONFIRMATIONS_NEEDED: usize = 11; + const PAID_PEER_OFFSET: u8 = 30; + + let key = xor_name_from_byte(0x62); + let config = ReplicationConfig::default(); + + let quorum_peers: Vec = + (1..=PRODUCTION_PAID_GROUP).map(peer_id_from_byte).collect(); + let paid_peers: Vec = (1..=PRODUCTION_PAID_GROUP) + .map(|i| peer_id_from_byte(PAID_PEER_OFFSET + i)) + .collect(); + let targets = single_key_targets(&key, quorum_peers.clone(), paid_peers.clone()); + + let presence = quorum_peers + .iter() + .enumerate() + .map(|(i, peer)| { + ( + *peer, + if i < STORAGE_HOLDERS_BELOW_QUORUM { + PresenceEvidence::Present + } else { + PresenceEvidence::Absent + }, + ) + }) + .collect(); + let paid_list = paid_peers + .iter() + .enumerate() + .map(|(i, peer)| { + ( + *peer, + if i < PAID_CONFIRMATIONS_NEEDED { + PaidListEvidence::Confirmed + } else { + PaidListEvidence::NotFound + }, + ) + }) + .collect(); + let evidence = build_evidence(presence, paid_list); + + let outcome = evaluate_key_evidence(&key, &evidence, &targets, &config); + + assert!( + matches!(outcome, KeyVerificationOutcome::PaidListVerified { ref sources } if sources.len() == STORAGE_HOLDERS_BELOW_QUORUM), + "11/20 paid-list confirmations must authorize repair despite only 3 storage holders, got {outcome:?}" ); } @@ -1229,6 +1734,139 @@ mod tests { ); } + #[test] + fn process_response_ignores_unsolicited_paid_status() { + let key = xor_name_from_byte(0xB2); + let peer = peer_id_from_byte(4); + + let targets = single_key_targets(&key, vec![peer], vec![]); + + let mut evidence: HashMap = std::iter::once(( + key, + KeyVerificationEvidence { + presence: HashMap::new(), + paid_list: HashMap::new(), + }, + )) + .collect(); + + let response = VerificationResponse { + results: vec![KeyVerificationResult { + key, + present: true, + paid: Some(true), + }], + }; + + process_verification_response(&peer, &response, &targets, &mut evidence); + + let ev = evidence.get(&key).expect("evidence for key"); + assert_eq!(ev.presence.get(&peer), Some(&PresenceEvidence::Present)); + assert!( + !ev.paid_list.contains_key(&peer), + "paid evidence must be recorded only for requested paid-list checks" + ); + } + + #[test] + fn process_batch_response_marks_only_batch_keys_unresolved() { + let key_a = xor_name_from_byte(0xB3); + let key_b = xor_name_from_byte(0xB4); + let peer = peer_id_from_byte(6); + + let targets = VerificationTargets { + quorum_targets: [(key_a, vec![peer]), (key_b, vec![peer])] + .into_iter() + .collect(), + paid_targets: [(key_a, vec![peer]), (key_b, vec![peer])] + .into_iter() + .collect(), + paid_group_sizes: [(key_a, 1), (key_b, 1)].into_iter().collect(), + paid_edge_targets: [ + (key_a, std::iter::once(peer).collect()), + (key_b, std::iter::once(peer).collect()), + ] + .into_iter() + .collect(), + all_peers: std::iter::once(peer).collect(), + peer_to_keys: std::iter::once((peer, vec![key_a, key_b])).collect(), + peer_to_paid_keys: std::iter::once(( + peer, + [key_a, key_b].into_iter().collect::>(), + )) + .collect(), + }; + let mut evidence: HashMap = [ + ( + key_a, + KeyVerificationEvidence { + presence: HashMap::new(), + paid_list: HashMap::new(), + }, + ), + ( + key_b, + KeyVerificationEvidence { + presence: HashMap::new(), + paid_list: HashMap::new(), + }, + ), + ] + .into_iter() + .collect(); + + process_verification_response_for_keys( + &peer, + &[key_a], + &VerificationResponse { + results: Vec::new(), + }, + &targets, + &mut evidence, + ); + + let ev_a = evidence.get(&key_a).expect("evidence for key_a"); + assert_eq!( + ev_a.presence.get(&peer), + Some(&PresenceEvidence::Unresolved) + ); + assert_eq!( + ev_a.paid_list.get(&peer), + Some(&PaidListEvidence::Unresolved) + ); + + let ev_b = evidence.get(&key_b).expect("evidence for key_b"); + assert!( + !ev_b.presence.contains_key(&peer), + "keys outside the failed batch must wait for their own batch result" + ); + assert!( + !ev_b.paid_list.contains_key(&peer), + "paid status outside the failed batch must not be prefilled" + ); + } + + #[test] + fn verification_requests_for_peer_splits_large_batches_and_rebases_paid_indices() { + let keys: Vec = (0..=MAX_VERIFICATION_KEYS_PER_REQUEST) + .map(xor_name_from_usize) + .collect(); + let paid_keys: HashSet = [keys[0], keys[MAX_VERIFICATION_KEYS_PER_REQUEST]] + .into_iter() + .collect(); + + let requests = verification_requests_for_peer(&keys, Some(&paid_keys)); + + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].keys.len(), MAX_VERIFICATION_KEYS_PER_REQUEST); + assert_eq!(requests[0].paid_list_check_indices, vec![0]); + assert_eq!( + requests[1].keys, + vec![keys[MAX_VERIFICATION_KEYS_PER_REQUEST]] + ); + assert_eq!(requests[1].paid_list_check_indices, vec![0]); + } + // ----------------------------------------------------------------------- // mark_peer_unresolved // ----------------------------------------------------------------------- @@ -1244,6 +1882,7 @@ mod tests { quorum_targets: std::iter::once((key_a, vec![peer])).collect(), paid_targets: std::iter::once((key_b, vec![peer])).collect(), paid_group_sizes: [(key_a, 0), (key_b, 1)].into_iter().collect(), + paid_edge_targets: std::iter::once((key_b, std::iter::once(peer).collect())).collect(), all_peers: std::iter::once(peer).collect(), peer_to_keys: std::iter::once((peer, vec![key_a, key_b])).collect(), peer_to_paid_keys: std::iter::once((peer, std::iter::once(key_b).collect())).collect(), @@ -1474,6 +2113,8 @@ mod tests { let mut paid_targets = HashMap::new(); let paid_group_size_a = paid_peers_a.len(); let paid_group_size_b = paid_peers_b.len(); + let paid_edge_targets_a = paid_edge_targets_for_peers(&paid_peers_a); + let paid_edge_targets_b = paid_edge_targets_for_peers(&paid_peers_b); paid_targets.insert(*key_a, paid_peers_a); paid_targets.insert(*key_b, paid_peers_b); @@ -1483,6 +2124,9 @@ mod tests { paid_group_sizes: [(*key_a, paid_group_size_a), (*key_b, paid_group_size_b)] .into_iter() .collect(), + paid_edge_targets: [(*key_a, paid_edge_targets_a), (*key_b, paid_edge_targets_b)] + .into_iter() + .collect(), all_peers, peer_to_keys, peer_to_paid_keys, @@ -1817,7 +2461,8 @@ mod tests { let paid_peers: Vec = (10..=14).map(peer_id_from_byte).collect(); let targets = single_key_targets(&key, quorum_peers.clone(), paid_peers.clone()); - // All quorum peers Absent; only 2/5 paid confirmations (below 3). + // All quorum peers Absent; only one paid confirmation, below the + // dynamic edge-aware paid-list threshold. let evidence = build_evidence( quorum_peers .iter() @@ -1825,7 +2470,7 @@ mod tests { .collect(), vec![ (paid_peers[0], PaidListEvidence::Confirmed), - (paid_peers[1], PaidListEvidence::Confirmed), + (paid_peers[1], PaidListEvidence::NotFound), (paid_peers[2], PaidListEvidence::NotFound), (paid_peers[3], PaidListEvidence::NotFound), (paid_peers[4], PaidListEvidence::NotFound), diff --git a/src/replication/scheduling.rs b/src/replication/scheduling.rs index ce02386c..cfaee6f3 100644 --- a/src/replication/scheduling.rs +++ b/src/replication/scheduling.rs @@ -121,6 +121,8 @@ pub struct InFlightEntry { pub all_sources: Vec, /// Sources already attempted (failed or in progress). pub tried: HashSet, + /// Pending-verification entry to restore if all fetch sources fail. + pub retry_verification: Option, } // --------------------------------------------------------------------------- @@ -133,6 +135,11 @@ pub struct InFlightEntry { /// 1. **`PendingVerify`** -- keys awaiting quorum verification. /// 2. **`FetchQueue`** -- quorum-passed keys waiting for a fetch slot. /// 3. **`InFlightFetch`** -- keys actively being downloaded. +/// +/// A key promoted from `PendingVerify` to fetch keeps a reserved verification +/// slot until it either stores successfully or returns to `PendingVerify`. +/// That reservation prevents unrelated new hints from stealing the capacity +/// needed to retry verification after every fetch source fails. pub struct ReplicationQueues { /// Keys awaiting quorum result (dedup by key). /// @@ -150,11 +157,13 @@ pub struct ReplicationQueues { in_flight_fetch: HashMap, /// Number of `pending_verify` entries currently attributed to each /// `hint_sender` peer. Maintained in lockstep with `pending_verify` - /// (insert/remove/evict) so the per-peer quota - /// ([`MAX_PENDING_VERIFY_PER_PEER`]) can be enforced in O(1). An entry is - /// removed from this map when its count reaches zero so the map itself is - /// bounded by the number of distinct currently-pending sources. + /// (insert/remove/evict). pending_per_sender: HashMap, + /// Pending-verification capacity slots reserved by retry-capable keys that + /// have left `pending_verify` for `fetch_queue` / `in_flight_fetch`. + retry_reserved_slots: usize, + /// Per-source view of [`Self::retry_reserved_slots`]. + retry_reserved_per_sender: HashMap, } impl Default for ReplicationQueues { @@ -173,6 +182,8 @@ impl ReplicationQueues { fetch_queue_keys: HashSet::new(), in_flight_fetch: HashMap::new(), pending_per_sender: HashMap::new(), + retry_reserved_slots: 0, + retry_reserved_per_sender: HashMap::new(), } } @@ -200,7 +211,7 @@ impl ReplicationQueues { if self.contains_key(&key) { return AdmissionResult::AlreadyPresent; } - if self.pending_verify.len() >= MAX_PENDING_VERIFY { + if self.pending_capacity_used() >= MAX_PENDING_VERIFY { debug!( "pending_verify at global capacity ({MAX_PENDING_VERIFY}); rejecting key {}", hex::encode(key) @@ -208,7 +219,7 @@ impl ReplicationQueues { return AdmissionResult::CapacityRejected; } let sender = entry.hint_sender; - let sender_count = self.pending_per_sender.get(&sender).copied().unwrap_or(0); + let sender_count = self.sender_capacity_used(&sender); if sender_count >= MAX_PENDING_VERIFY_PER_PEER { debug!( "peer {sender} at per-source pending cap ({MAX_PENDING_VERIFY_PER_PEER}); \ @@ -217,11 +228,29 @@ impl ReplicationQueues { ); return AdmissionResult::CapacityRejected; } - self.pending_verify.insert(key, entry); - *self.pending_per_sender.entry(sender).or_insert(0) += 1; + self.insert_pending_unchecked(key, entry); AdmissionResult::Admitted } + fn pending_capacity_used(&self) -> usize { + self.pending_verify + .len() + .saturating_add(self.retry_reserved_slots) + } + + fn sender_capacity_used(&self, sender: &PeerId) -> usize { + self.pending_per_sender + .get(sender) + .copied() + .unwrap_or(0) + .saturating_add( + self.retry_reserved_per_sender + .get(sender) + .copied() + .unwrap_or(0), + ) + } + /// Decrement (and prune at zero) the per-sender counter for `sender`. /// /// Kept private so the counter can only move in lockstep with @@ -239,6 +268,42 @@ impl ReplicationQueues { } } + fn insert_pending_unchecked(&mut self, key: XorName, entry: VerificationEntry) { + let sender = entry.hint_sender; + let replaced = self.pending_verify.insert(key, entry); + debug_assert!( + replaced.is_none(), + "pending entry inserted twice for {}", + hex::encode(key) + ); + *self.pending_per_sender.entry(sender).or_insert(0) += 1; + } + + fn reserve_retry_slot(&mut self, sender: PeerId) { + self.retry_reserved_slots = self.retry_reserved_slots.saturating_add(1); + *self.retry_reserved_per_sender.entry(sender).or_insert(0) += 1; + } + + fn release_retry_slot(&mut self, sender: &PeerId) { + if !self.retry_reserved_per_sender.contains_key(sender) { + return; + } + self.retry_reserved_slots = self.retry_reserved_slots.saturating_sub(1); + Self::release_sender_slot(&mut self.retry_reserved_per_sender, sender); + } + + fn release_retry_slot_for_entry(&mut self, entry: &InFlightEntry) { + if let Some(verification) = &entry.retry_verification { + self.release_retry_slot(&verification.hint_sender); + } + } + + fn release_retry_slot_for_candidate(&mut self, candidate: &FetchCandidate) { + if let Some(verification) = &candidate.retry_verification { + self.release_retry_slot(&verification.hint_sender); + } + } + /// Get a reference to a pending verification entry. #[must_use] pub fn get_pending(&self, key: &XorName) -> Option<&VerificationEntry> { @@ -281,6 +346,24 @@ impl ReplicationQueues { self.pending_verify.keys().copied().collect() } + /// Collect pending verification keys whose retry delay has elapsed. + #[must_use] + pub fn ready_pending_keys(&self, now: Instant) -> Vec { + self.pending_verify + .iter() + .filter_map(|(key, entry)| (entry.next_verify_at <= now).then_some(*key)) + .collect() + } + + /// Defer a pending key before its next verification attempt. + pub fn defer_pending(&mut self, key: &XorName, retry_after: Duration) -> bool { + let Some(entry) = self.pending_verify.get_mut(key) else { + return false; + }; + entry.next_verify_at = Instant::now() + retry_after; + true + } + /// Number of keys in pending verification. #[must_use] pub fn pending_count(&self) -> usize { @@ -303,6 +386,29 @@ impl ReplicationQueues { /// place when the fetch queue is full (so verified work is retried on /// the next cycle instead of being silently lost). pub fn enqueue_fetch(&mut self, key: XorName, distance: XorName, sources: Vec) -> bool { + if self.pending_verify.contains_key(&key) + || self.fetch_queue_keys.contains(&key) + || self.in_flight_fetch.contains_key(&key) + { + return false; + } + if self.fetch_queue.len() >= MAX_FETCH_QUEUE { + debug!( + "fetch_queue at capacity ({MAX_FETCH_QUEUE}); dropping new key {}", + hex::encode(key) + ); + return false; + } + self.enqueue_fetch_with_retry(key, distance, sources, None) + } + + fn enqueue_fetch_with_retry( + &mut self, + key: XorName, + distance: XorName, + sources: Vec, + retry_verification: Option, + ) -> bool { if self.pending_verify.contains_key(&key) || self.fetch_queue_keys.contains(&key) || self.in_flight_fetch.contains_key(&key) @@ -321,6 +427,7 @@ impl ReplicationQueues { key, distance, sources, + retry_verification, }); true } @@ -351,12 +458,24 @@ impl ReplicationQueues { return false; } // Capacity confirmed; safe to release the pending slot and enqueue. - let _ = self.remove_pending(&key); + let retry_verification = self.remove_pending(&key); + let retry_sender = retry_verification + .as_ref() + .map(|verification| verification.hint_sender); + if let Some(sender) = retry_sender { + self.reserve_retry_slot(sender); + } // enqueue_fetch returns false only on capacity or already-queued; the // capacity check above and the just-removed pending state make this // succeed. If a concurrent path put the key into fetch_queue/in_flight // between, dropping the duplicate is fine. - self.enqueue_fetch(key, distance, sources) + let enqueued = self.enqueue_fetch_with_retry(key, distance, sources, retry_verification); + if !enqueued { + if let Some(sender) = retry_sender { + self.release_retry_slot(&sender); + } + } + enqueued } /// Dequeue the nearest fetch candidate. @@ -364,12 +483,19 @@ impl ReplicationQueues { /// Returns `None` when the queue is empty. Silently skips candidates /// that are somehow already in-flight. Concurrency is enforced by the /// fetch worker, not by this method. + /// + /// A returned candidate may carry a live verification retry-slot + /// reservation. Callers must consume it with + /// [`Self::start_dequeued_fetch`], [`Self::discard_fetch_candidate`], or + /// [`Self::requeue_candidate_for_verification`] so that reservation is + /// either transferred, released, or restored to `pending_verify`. pub fn dequeue_fetch(&mut self) -> Option { while let Some(candidate) = self.fetch_queue.pop() { self.fetch_queue_keys.remove(&candidate.key); if !self.in_flight_fetch.contains_key(&candidate.key) { return Some(candidate); } + self.release_retry_slot_for_candidate(&candidate); } None } @@ -385,10 +511,32 @@ impl ReplicationQueues { // ----------------------------------------------------------------------- /// Mark a key as in-flight (actively being fetched from `source`). + /// + /// Candidates returned by [`Self::dequeue_fetch`] MUST be consumed by a + /// by-value dequeued-candidate method instead. They may carry a live + /// verification retry-slot reservation; [`Self::start_dequeued_fetch`] + /// transfers that reservation into the in-flight entry. pub fn start_fetch(&mut self, key: XorName, source: PeerId, all_sources: Vec) { + self.start_fetch_with_retry(key, source, all_sources, None); + } + + /// Mark a key as in-flight and retain verification retry metadata. + /// + /// This is for direct starts where the caller already owns any retry + /// reservation paired with `retry_verification`. Candidates obtained from + /// [`Self::dequeue_fetch`] MUST be consumed intact via a by-value + /// dequeued-candidate method, otherwise their reserved verification + /// capacity can be orphaned. + pub fn start_fetch_with_retry( + &mut self, + key: XorName, + source: PeerId, + all_sources: Vec, + retry_verification: Option, + ) { let mut tried = HashSet::new(); tried.insert(source); - self.in_flight_fetch.insert( + let replaced = self.in_flight_fetch.insert( key, InFlightEntry { key, @@ -396,13 +544,43 @@ impl ReplicationQueues { started_at: Instant::now(), all_sources, tried, + retry_verification, }, ); + if let Some(entry) = replaced { + self.release_retry_slot_for_entry(&entry); + } + } + + /// Consume a dequeued fetch candidate and transfer its retry reservation + /// into the in-flight entry. + pub fn start_dequeued_fetch(&mut self, candidate: FetchCandidate, source: PeerId) { + let FetchCandidate { + key, + sources, + retry_verification, + .. + } = candidate; + self.start_fetch_with_retry(key, source, sources, retry_verification); } /// Mark a fetch as completed (success or permanent failure). pub fn complete_fetch(&mut self, key: &XorName) -> Option { - self.in_flight_fetch.remove(key) + let removed = self.in_flight_fetch.remove(key); + if let Some(entry) = &removed { + self.release_retry_slot_for_entry(entry); + } + removed + } + + /// Drop a dequeued fetch candidate without starting it. + pub fn discard_fetch_candidate(&mut self, candidate: FetchCandidate) { + let FetchCandidate { + retry_verification, .. + } = candidate; + if let Some(verification) = retry_verification { + self.release_retry_slot(&verification.hint_sender); + } } /// Mark the current fetch attempt as failed and try the next untried source. @@ -428,6 +606,54 @@ impl ReplicationQueues { } } + /// Consume a dequeued candidate and restore its verification entry for a + /// later retry when retry metadata exists. + pub fn requeue_candidate_for_verification( + &mut self, + candidate: FetchCandidate, + retry_after: Duration, + ) -> bool { + let FetchCandidate { + key, + retry_verification, + .. + } = candidate; + let Some(mut verification) = retry_verification else { + return false; + }; + let sender = verification.hint_sender; + + verification.state = VerificationState::PendingVerify; + verification.verified_sources.clear(); + verification.tried_sources.clear(); + verification.next_verify_at = Instant::now() + retry_after; + + self.insert_pending_unchecked(key, verification); + self.release_retry_slot(&sender); + true + } + + /// Complete an exhausted fetch and restore its verification entry for a + /// later retry when retry metadata exists. + pub fn requeue_fetch_for_verification(&mut self, key: &XorName, retry_after: Duration) -> bool { + let Some(mut entry) = self.in_flight_fetch.remove(key) else { + return false; + }; + let Some(mut verification) = entry.retry_verification.take() else { + return false; + }; + let sender = verification.hint_sender; + + verification.state = VerificationState::PendingVerify; + verification.verified_sources.clear(); + verification.tried_sources.clear(); + verification.next_verify_at = Instant::now() + retry_after; + + self.insert_pending_unchecked(*key, verification); + self.release_retry_slot(&sender); + true + } + /// Number of in-flight fetches. #[must_use] pub fn in_flight_count(&self) -> usize { @@ -455,28 +681,39 @@ impl ReplicationQueues { } /// Evict stale pending-verification entries older than `max_age`. - pub fn evict_stale(&mut self, max_age: Duration) { + pub fn evict_stale(&mut self, max_age: Duration) -> Vec { let now = Instant::now(); - let before = self.pending_verify.len(); - let pending_per_sender = &mut self.pending_per_sender; - self.pending_verify.retain(|_, entry| { - let fresh = now.duration_since(entry.created_at) < max_age; - if !fresh { - Self::release_sender_slot(pending_per_sender, &entry.hint_sender); + let evicted_keys = self + .pending_verify + .iter() + .filter_map(|(key, entry)| { + (now.duration_since(entry.created_at) >= max_age).then_some(*key) + }) + .collect::>(); + + for key in &evicted_keys { + if let Some(entry) = self.pending_verify.remove(key) { + Self::release_sender_slot(&mut self.pending_per_sender, &entry.hint_sender); } - fresh - }); - let evicted = before.saturating_sub(self.pending_verify.len()); - if evicted > 0 { - debug!("Evicted {evicted} stale pending-verification entries"); } + + if !evicted_keys.is_empty() { + debug!( + "Evicted {} stale pending-verification entries", + evicted_keys.len() + ); + } + + evicted_keys } /// Number of `pending_verify` entries currently attributed to `sender`. - /// Exposed for tests and observability of the per-source fairness quota. + /// Includes retry reservations held by verified keys currently in the fetch + /// pipeline, because those reservations still consume the sender's fairness + /// quota. #[must_use] pub fn pending_count_for_sender(&self, sender: &PeerId) -> usize { - self.pending_per_sender.get(sender).copied().unwrap_or(0) + self.sender_capacity_used(sender) } } @@ -504,18 +741,131 @@ mod tests { [b; 32] } + fn xor_name_from_u32(value: u32) -> XorName { + let mut name = [0u8; 32]; + name[..4].copy_from_slice(&value.to_le_bytes()); + name + } + /// Create a minimal `VerificationEntry` for testing. fn test_entry(sender_byte: u8) -> VerificationEntry { + let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: peer_id_from_byte(sender_byte), } } + struct ReservedCandidateAtSenderCap { + queues: ReplicationQueues, + key: XorName, + source: PeerId, + hint_sender: PeerId, + fresh_key: XorName, + candidate: FetchCandidate, + base_sender_count: usize, + pre_promotion_sender_count: usize, + } + + fn assert_sender_cap_rejects_key( + queues: &mut ReplicationQueues, + key: XorName, + sender_byte: u8, + ) { + assert_eq!( + queues.add_pending_verify(key, test_entry(sender_byte)), + AdmissionResult::CapacityRejected, + "fresh key should be rejected while sender capacity is exhausted" + ); + } + + fn assert_sender_released_slot_admits_key( + queues: &mut ReplicationQueues, + sender: &PeerId, + sender_byte: u8, + key: XorName, + expected_count_before_admission: usize, + ) { + assert_eq!( + queues.pending_count_for_sender(sender), + expected_count_before_admission, + "sender capacity should return to the expected count" + ); + assert!( + queues + .add_pending_verify(key, test_entry(sender_byte)) + .admitted(), + "fresh key should be admitted after a retry reservation is released" + ); + assert_eq!( + queues.pending_count_for_sender(sender), + expected_count_before_admission + 1, + "fresh key should consume the released sender slot" + ); + } + + fn reserved_candidate_at_sender_cap() -> ReservedCandidateAtSenderCap { + const PROMOTED_KEY_INDEX: u32 = 40_000; + const FILLER_KEY_OFFSET: u32 = 50_000; + const FRESH_KEY_INDEX: u32 = 60_000; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const HINT_SENDER_BYTE: u8 = 9; + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_u32(PROMOTED_KEY_INDEX); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let fresh_key = xor_name_from_u32(FRESH_KEY_INDEX); + let base_sender_count = MAX_PENDING_VERIFY_PER_PEER - 1; + let pre_promotion_sender_count = MAX_PENDING_VERIFY_PER_PEER; + + assert!(queues + .add_pending_verify(key, test_entry(HINT_SENDER_BYTE)) + .admitted()); + for i in 0..base_sender_count { + let key_index = FILLER_KEY_OFFSET + u32::try_from(i).expect("test index fits u32"); + assert!( + queues + .add_pending_verify(xor_name_from_u32(key_index), test_entry(HINT_SENDER_BYTE)) + .admitted(), + "filler key should be admitted before the sender reaches its cap" + ); + } + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "sender should be exactly at capacity before promotion" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + + assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "promoted candidate should retain its sender capacity reservation" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + ReservedCandidateAtSenderCap { + queues, + key, + source, + hint_sender, + fresh_key, + candidate, + base_sender_count, + pre_promotion_sender_count, + } + } + // -- add_pending_verify dedup ------------------------------------------ #[test] @@ -577,9 +927,11 @@ mod tests { let first = queues.dequeue_fetch().expect("should dequeue"); assert_eq!(first.key, near_key, "nearest key should dequeue first"); + queues.discard_fetch_candidate(first); let second = queues.dequeue_fetch().expect("should dequeue"); assert_eq!(second.key, far_key, "farthest key should dequeue second"); + queues.discard_fetch_candidate(second); } #[test] @@ -651,6 +1003,317 @@ mod tests { assert!(queues.retry_fetch(&xor_name_from_byte(0xFF)).is_none()); } + #[test] + fn exhausted_promoted_fetch_requeues_verification() { + const KEY_BYTE: u8 = 0x44; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(KEY_BYTE); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.hint_sender = hint_sender; + + assert!(queues.add_pending_verify(key, entry).admitted()); + assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + queues.start_dequeued_fetch(candidate, source); + + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!(queues.requeue_fetch_for_verification(&key, RETRY_DELAY)); + + assert_eq!(queues.in_flight_count(), 0); + assert_eq!(queues.pending_count_for_sender(&hint_sender), 1); + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "requeued key should observe retry delay" + ); + + let after_retry = Instant::now() + RETRY_DELAY + RETRY_DELAY_SLACK; + assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); + } + + #[test] + fn promoted_fetch_reserves_sender_capacity_for_requeue() { + const PROMOTED_KEY_INDEX: u32 = 10_000; + const EXTRA_KEY_OFFSET: u32 = 20_000; + const REJECTED_KEY_INDEX: u32 = 30_000; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_u32(PROMOTED_KEY_INDEX); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.hint_sender = hint_sender; + + assert!(queues.add_pending_verify(key, entry).admitted()); + assert!(queues.promote_pending_to_fetch(key, distance, vec![source])); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + 1, + "fetch candidate must retain its sender quota reservation" + ); + + for i in 0..(MAX_PENDING_VERIFY_PER_PEER - 1) { + let key_index = EXTRA_KEY_OFFSET + u32::try_from(i).expect("test index fits u32"); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.hint_sender = hint_sender; + assert!( + queues + .add_pending_verify(xor_name_from_u32(key_index), entry) + .admitted(), + "sender should admit up to the quota not including the reserved fetch slot" + ); + } + + let mut rejected_entry = test_entry(HINT_SENDER_BYTE); + rejected_entry.hint_sender = hint_sender; + assert_eq!( + queues.add_pending_verify(xor_name_from_u32(REJECTED_KEY_INDEX), rejected_entry), + AdmissionResult::CapacityRejected, + "reserved fetch slot must count toward the per-sender capacity" + ); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + queues.start_dequeued_fetch(candidate, source); + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!( + queues.requeue_fetch_for_verification(&key, RETRY_DELAY), + "requeue must use the reserved slot even while the sender is at capacity" + ); + + assert!(queues.get_pending(&key).is_some()); + assert_eq!(queues.pending_count(), MAX_PENDING_VERIFY_PER_PEER); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + MAX_PENDING_VERIFY_PER_PEER + ); + } + + #[test] + fn start_dequeued_fetch_then_complete_releases_reserved_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + + let ReservedCandidateAtSenderCap { + mut queues, + key, + source, + hint_sender, + fresh_key, + candidate, + base_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + queues.start_dequeued_fetch(candidate, source); + assert!(queues.complete_fetch(&key).is_some()); + + assert_sender_released_slot_admits_key( + &mut queues, + &hint_sender, + HINT_SENDER_BYTE, + fresh_key, + base_sender_count, + ); + } + + #[test] + fn start_dequeued_fetch_then_exhaust_requeues_with_reserved_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let ReservedCandidateAtSenderCap { + mut queues, + key, + source, + hint_sender, + fresh_key, + candidate, + pre_promotion_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + queues.start_dequeued_fetch(candidate, source); + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!( + queues.requeue_fetch_for_verification(&key, RETRY_DELAY), + "exhausted fetch should restore retry metadata" + ); + + assert!(queues.get_pending(&key).is_some()); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "requeued key should convert its reservation back to a pending slot" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + } + + #[test] + fn discard_dequeued_fetch_candidate_releases_reserved_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + + let ReservedCandidateAtSenderCap { + mut queues, + hint_sender, + fresh_key, + candidate, + base_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + queues.discard_fetch_candidate(candidate); + + assert_sender_released_slot_admits_key( + &mut queues, + &hint_sender, + HINT_SENDER_BYTE, + fresh_key, + base_sender_count, + ); + } + + #[test] + fn requeue_dequeued_fetch_candidate_restores_pending_sender_capacity() { + const HINT_SENDER_BYTE: u8 = 9; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let ReservedCandidateAtSenderCap { + mut queues, + key, + hint_sender, + fresh_key, + candidate, + pre_promotion_sender_count, + .. + } = reserved_candidate_at_sender_cap(); + + assert!( + queues.requeue_candidate_for_verification(candidate, RETRY_DELAY), + "dequeued retry candidate should be restored to pending verification" + ); + + assert!(queues.get_pending(&key).is_some()); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + pre_promotion_sender_count, + "requeued candidate should convert its reservation back to a pending slot" + ); + assert_sender_cap_rejects_key(&mut queues, fresh_key, HINT_SENDER_BYTE); + } + + #[test] + fn no_sources_dequeued_candidate_requeues_for_verification() { + const KEY_INDEX: u32 = 70_000; + const DISTANCE_BYTE: u8 = 0x01; + const HINT_SENDER_BYTE: u8 = 9; + const VERIFIED_SOURCE_BYTE: u8 = 2; + const TRIED_SOURCE_BYTE: u8 = 3; + const RETRY_DELAY: Duration = Duration::from_secs(15); + const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); + const REQUEUED_SENDER_COUNT: usize = 1; + const EMPTY_SENDER_COUNT: usize = 0; + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_u32(KEY_INDEX); + let distance = xor_name_from_byte(DISTANCE_BYTE); + let hint_sender = peer_id_from_byte(HINT_SENDER_BYTE); + let verified_source = peer_id_from_byte(VERIFIED_SOURCE_BYTE); + let tried_source = peer_id_from_byte(TRIED_SOURCE_BYTE); + let mut entry = test_entry(HINT_SENDER_BYTE); + entry.state = VerificationState::QueuedForFetch; + entry.verified_sources.push(verified_source); + entry.tried_sources.insert(tried_source); + + assert!(queues.add_pending_verify(key, entry).admitted()); + assert!(queues.promote_pending_to_fetch(key, distance, Vec::new())); + + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + assert!( + candidate.sources.is_empty(), + "test candidate should exercise the no-sources branch" + ); + assert!( + queues.requeue_candidate_for_verification(candidate, RETRY_DELAY), + "no-sources retry candidate should be restored to pending verification" + ); + + let pending = queues.get_pending(&key).expect("key should be pending"); + assert_eq!(pending.state, VerificationState::PendingVerify); + assert!( + pending.verified_sources.is_empty(), + "verified sources should be cleared before retry" + ); + assert!( + pending.tried_sources.is_empty(), + "tried sources should be cleared before retry" + ); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + REQUEUED_SENDER_COUNT, + "retry reservation should be converted back to one pending slot" + ); + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "requeued key should observe retry delay" + ); + + let after_retry = Instant::now() + RETRY_DELAY + RETRY_DELAY_SLACK; + assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); + + assert!(queues.remove_pending(&key).is_some()); + assert_eq!( + queues.pending_count_for_sender(&hint_sender), + EMPTY_SENDER_COUNT, + "removing the requeued entry should leave no reserved sender slot" + ); + } + + #[test] + fn exhausted_direct_fetch_remains_terminal() { + const KEY_BYTE: u8 = 0x45; + const DISTANCE_BYTE: u8 = 0x01; + const SOURCE_BYTE: u8 = 2; + const RETRY_DELAY: Duration = Duration::from_secs(15); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(KEY_BYTE); + let source = peer_id_from_byte(SOURCE_BYTE); + + queues.enqueue_fetch(key, xor_name_from_byte(DISTANCE_BYTE), vec![source]); + let candidate = queues.dequeue_fetch().expect("fetch candidate"); + queues.start_dequeued_fetch(candidate, source); + + assert!( + queues.retry_fetch(&key).is_none(), + "single source should be exhausted" + ); + assert!(!queues.requeue_fetch_for_verification(&key, RETRY_DELAY)); + assert_eq!(queues.in_flight_count(), 0); + assert_eq!(queues.pending_count(), 0); + } + // -- contains_key across pipelines ------------------------------------ #[test] @@ -726,7 +1389,8 @@ mod tests { assert_eq!(queues.pending_count(), 1); assert_eq!(queues.pending_count_for_sender(&sender), 1); - queues.evict_stale(Duration::from_secs(1)); + let evicted = queues.evict_stale(Duration::from_secs(1)); + assert_eq!(evicted, vec![key]); assert_eq!( queues.pending_count(), 0, @@ -746,7 +1410,11 @@ mod tests { let key = xor_name_from_byte(0x01); queues.add_pending_verify(key, test_entry(1)); - queues.evict_stale(Duration::from_secs(3600)); + let evicted = queues.evict_stale(Duration::from_secs(3600)); + assert!( + evicted.is_empty(), + "fresh entry should not be reported as evicted" + ); assert_eq!( queues.pending_count(), 1, @@ -754,6 +1422,26 @@ mod tests { ); } + #[test] + fn deferred_pending_key_is_not_ready_until_retry_time() { + const RETRY_DELAY: Duration = Duration::from_secs(15); + const RETRY_DELAY_SLACK: Duration = Duration::from_secs(1); + + let mut queues = ReplicationQueues::new(); + let key = xor_name_from_byte(0xAA); + queues.add_pending_verify(key, test_entry(1)); + + assert_eq!(queues.ready_pending_keys(Instant::now()), vec![key]); + assert!(queues.defer_pending(&key, RETRY_DELAY)); + assert!( + queues.ready_pending_keys(Instant::now()).is_empty(), + "deferred key should not be retried immediately" + ); + + let after_retry = Instant::now() + RETRY_DELAY + RETRY_DELAY_SLACK; + assert_eq!(queues.ready_pending_keys(after_retry), vec![key]); + } + // -- remove_pending --------------------------------------------------- #[test] @@ -817,11 +1505,8 @@ mod tests { // Step 5: Dequeue, start fetch -> key is in-flight. let candidate = queues.dequeue_fetch().expect("should dequeue"); - queues.start_fetch( - candidate.key, - candidate.sources[0], - candidate.sources.clone(), - ); + let source = candidate.sources[0]; + queues.start_dequeued_fetch(candidate, source); // Step 6: Attempt to add to PendingVerify while in-flight -> reject. assert!( @@ -855,6 +1540,7 @@ mod tests { verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), + next_verify_at: Instant::now(), hint_sender: peer_id_from_byte(1), }; @@ -874,6 +1560,7 @@ mod tests { verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), + next_verify_at: Instant::now(), hint_sender: peer_id_from_byte(2), }; @@ -914,6 +1601,7 @@ mod tests { verified_sources: Vec::new(), tried_sources: HashSet::new(), created_at: Instant::now(), + next_verify_at: Instant::now(), hint_sender, }; assert!( @@ -940,7 +1628,7 @@ mod tests { let candidate = queues.dequeue_fetch().expect("should dequeue"); assert_eq!(candidate.key, key); assert_eq!(candidate.sources.len(), 2); - queues.start_fetch(key, source_a, candidate.sources); + queues.start_dequeued_fetch(candidate, source_a); assert_eq!(queues.in_flight_count(), 1); assert_eq!(queues.fetch_queue_count(), 0); assert!( diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index 9923a3a5..33a0f81b 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -782,6 +782,7 @@ fn failed( summary, reason, }, + no_response_class: None, } } diff --git a/src/replication/types.rs b/src/replication/types.rs index 59e6b417..bdbdf657 100644 --- a/src/replication/types.rs +++ b/src/replication/types.rs @@ -92,6 +92,9 @@ pub struct VerificationEntry { pub tried_sources: HashSet, /// When this entry was created. pub created_at: Instant, + /// Earliest time this key should be included in another verification + /// round. + pub next_verify_at: Instant, /// The peer that originally hinted this key (for source tracking). pub hint_sender: PeerId, } @@ -113,6 +116,9 @@ pub struct FetchCandidate { pub distance: XorName, /// Verified source peers that responded `Present`. pub sources: Vec, + /// Pending-verification entry to restore if every fetch source is + /// exhausted before the chunk is recovered. + pub retry_verification: Option, } impl Eq for FetchCandidate {} @@ -627,6 +633,17 @@ impl NeighborSyncState { pub fn is_cycle_complete(&self) -> bool { self.priority_order.is_empty() && self.cursor >= self.order.len() } + + /// Whether topology-change (priority) peers are still queued. + /// + /// The neighbor-sync loop drains these back-to-back rather than parking on + /// the periodic tick, so a churn burst converges at round-trip speed. The + /// `sync_trigger` `Notify` coalesces multiple wakeups into one, so it cannot + /// be the sole signal for pending work — this queue is the source of truth. + #[must_use] + pub fn has_priority_peers(&self) -> bool { + !self.priority_order.is_empty() + } } // --------------------------------------------------------------------------- @@ -731,6 +748,7 @@ mod tests { 0, 0, 0, 0, ], sources: vec![peer_id_from_byte(1)], + retry_verification: None, }; let far = FetchCandidate { @@ -740,6 +758,7 @@ mod tests { 0, 0, 0, 0, 0, ], sources: vec![peer_id_from_byte(2)], + retry_verification: None, }; // In a max-heap the "greatest" element pops first. @@ -775,12 +794,14 @@ mod tests { key: [1u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; let b = FetchCandidate { key: [1u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; assert_eq!( @@ -797,12 +818,14 @@ mod tests { key: [1u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; let b = FetchCandidate { key: [2u8; 32], distance: [5u8; 32], sources: vec![], + retry_verification: None, }; assert_ne!( @@ -1221,6 +1244,31 @@ mod tests { assert_eq!(state.priority_order.len(), 1); } + #[test] + fn neighbor_sync_has_priority_peers_tracks_queue_and_drain() { + // The neighbor-sync loop drains the priority queue back-to-back and only + // parks once `has_priority_peers` reports false. Draining removes each + // queued peer (as `select_next_sync_peer` does via `remove_peer`), so the + // signal must flip to false once every entrant is consumed — this is the + // loop's termination guarantee. + let first = peer_id_from_byte(6); + let second = peer_id_from_byte(7); + let mut state = NeighborSyncState::new_cycle(Vec::new()); + assert!(!state.has_priority_peers()); + + assert_eq!(state.queue_priority_peers([first, second]), 2); + assert!(state.has_priority_peers()); + + assert!(state.remove_peer(&first)); + assert!(state.has_priority_peers(), "one entrant still pending"); + + assert!(state.remove_peer(&second)); + assert!( + !state.has_priority_peers(), + "drained queue must let the loop park" + ); + } + #[test] fn neighbor_sync_remove_peer_clears_order_and_priority_queue() { let peer = peer_id_from_byte(4); diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index df9e05a6..16976ac2 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -8,6 +8,7 @@ use super::testnet::TestNetworkConfig; use super::TestHarness; use ant_node::client::compute_address; +use ant_node::replication::audit_coordinator::AuditChallengeCoordinator; use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; use ant_node::replication::config::{ storage_admission_width, K_BUCKET_SIZE, REPAIR_HINT_MIN_AGE, REPLICATION_PROTOCOL_ID, @@ -47,6 +48,28 @@ const FULL_NODE_SHUN_POSSESSION_DELAY_MAX: Duration = Duration::from_millis(500) const DUMMY_PAYMENT_PROOF_LEN: usize = 64; /// Dummy proof byte used when a test only needs to reach pre-payment gates. const DUMMY_PAYMENT_PROOF_BYTE: u8 = 0x01; +/// Minimal paid-list repair close group used by the deterministic repair e2e. +const PAID_REPAIR_GROUP_SIZE: usize = 5; +/// Storage threshold configured above majority so one holder is below quorum. +const PAID_REPAIR_STORAGE_THRESHOLD: usize = 4; +/// Paid-list majority for a five-peer group. +const PAID_REPAIR_CONFIRMING_NODES: usize = 3; +/// Single node seeded with the record bytes before repair. +const PAID_REPAIR_SOURCE_INDEX: usize = 0; +/// Missing responsible node that must learn the paid-list entry and fetch. +const PAID_REPAIR_TARGET_INDEX: usize = 4; +/// Expected storage quorum for the five-peer repair group. +const PAID_REPAIR_STORAGE_QUORUM: usize = 3; +/// Timeout used by the repair e2e's verification requests. +const PAID_REPAIR_VERIFICATION_TIMEOUT: Duration = Duration::from_secs(3); +/// Timeout used by the repair e2e's fetch requests. +const PAID_REPAIR_FETCH_TIMEOUT: Duration = Duration::from_secs(3); +/// Wait budget for asynchronous verification plus fetch completion. +const PAID_REPAIR_SETTLE_TIMEOUT: Duration = Duration::from_secs(30); +/// Request-response timeout for seeding the replica hint. +const PAID_REPAIR_HINT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +/// Stable request id for the paid-list repair sync request. +const PAID_REPAIR_HINT_REQUEST_ID: u64 = 2526; /// Send a replication request via saorsa-core's request-response mechanism /// and decode the response. @@ -955,6 +978,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { let config = prune_test_config(close_group_size); let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1000,6 +1024,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proof_now: Some(gate_repair_proof_now), allow_remote_prune_audits: false, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(blocked.records_pruned, 0); @@ -1020,6 +1045,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proof_now: Some(gate_repair_proof_now), allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(confirmed.records_pruned, 1); @@ -1065,6 +1091,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proof_now: Some(missing_repair_proof_now), allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(incomplete.records_pruned, 0); @@ -1093,6 +1120,7 @@ async fn test_prune_pass_requires_remote_confirmation_before_delete() { repair_proof_now: Some(missing_repair_proof_now), allow_remote_prune_audits: true, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!(complete.records_pruned, 1); @@ -1126,6 +1154,7 @@ async fn test_prune_veto_for_committed_out_of_range_key() { let config = prune_test_config(close_group_size); let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1195,6 +1224,7 @@ async fn test_prune_veto_for_committed_out_of_range_key() { allow_remote_prune_audits: true, repair_proof_now: Some(repair_proof_now), commitment_state: Some(&committed), + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1220,6 +1250,7 @@ async fn test_prune_veto_for_committed_out_of_range_key() { allow_remote_prune_audits: true, repair_proof_now: Some(repair_proof_now), commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1277,6 +1308,7 @@ async fn prune_deletes_at_proof_threshold_and_retains_below_it() { }; let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1335,6 +1367,7 @@ async fn prune_deletes_at_proof_threshold_and_retains_below_it() { allow_remote_prune_audits: true, repair_proof_now: Some(repair_proof_now), commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1371,6 +1404,7 @@ async fn prune_deletes_at_proof_threshold_and_retains_below_it() { allow_remote_prune_audits: true, repair_proof_now: Some(repair_proof_now), commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1412,6 +1446,7 @@ async fn paid_prune_requires_paid_close_group_confirmations() { }; let sync_state = Arc::new(RwLock::new(NeighborSyncState::new_cycle(vec![]))); let repair_proofs = Arc::new(RwLock::new(RepairProofs::new())); + let audit_challenge_coordinator = Arc::new(AuditChallengeCoordinator::new()); let pruner = harness.test_node(pruner_idx).expect("pruner"); let pruner_p2p = Arc::clone(pruner.p2p_node.as_ref().expect("pruner p2p")); @@ -1445,6 +1480,7 @@ async fn paid_prune_requires_paid_close_group_confirmations() { allow_remote_prune_audits: true, repair_proof_now: None, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1481,6 +1517,7 @@ async fn paid_prune_requires_paid_close_group_confirmations() { allow_remote_prune_audits: true, repair_proof_now: None, commitment_state: None, + audit_challenge_coordinator: &audit_challenge_coordinator, }) .await; assert_eq!( @@ -1967,7 +2004,7 @@ async fn scenario_9_fetch_retry_uses_alternate_source() { let candidate = queues.dequeue_fetch().expect("dequeue"); // Start in-flight with first source - queues.start_fetch(key, source_a, candidate.sources); + queues.start_dequeued_fetch(candidate, source_a); // First source fails -> retry should give source_b let next = queues.retry_fetch(&key); @@ -1994,8 +2031,8 @@ async fn scenario_10_fetch_retry_exhaustion() { // Single source queues.enqueue_fetch(key, distance, vec![source]); - let _candidate = queues.dequeue_fetch().expect("dequeue"); - queues.start_fetch(key, source, vec![source]); + let candidate = queues.dequeue_fetch().expect("dequeue"); + queues.start_dequeued_fetch(candidate, source); // Source fails -> no alternates -> exhausted let next = queues.retry_fetch(&key); @@ -2554,6 +2591,198 @@ async fn scenario_25_paid_list_convergence_via_verification() { harness.teardown().await.expect("teardown"); } +// ========================================================================= +// Section 18, Scenario #26: Paid-list majority authorises repair +// ========================================================================= + +/// A missing responsible replica is repaired when storage presence is below +/// quorum but the paid-list close group still has a majority (Section 18 #26). +/// +/// This drives the live path end-to-end: +/// 1. one peer stores the bytes, which is below storage quorum; +/// 2. three of five paid-list peers confirm the key; +/// 3. the holder sends a replica hint to a missing responsible peer; +/// 4. verification learns paid-list authorization and fetches the record. +#[tokio::test] +#[serial] +async fn scenario_26_paid_list_majority_repairs_missing_replica_below_storage_quorum() { + let mut net_config = TestNetworkConfig::minimal(); + net_config.replication_config = Some(ReplicationConfig { + close_group_size: PAID_REPAIR_GROUP_SIZE, + quorum_threshold: PAID_REPAIR_STORAGE_THRESHOLD, + paid_list_close_group_size: PAID_REPAIR_GROUP_SIZE, + verification_request_timeout: PAID_REPAIR_VERIFICATION_TIMEOUT, + fetch_request_timeout: PAID_REPAIR_FETCH_TIMEOUT, + ..ReplicationConfig::default() + }); + + let harness = TestHarness::setup_with_config(net_config) + .await + .expect("setup"); + harness.warmup_dht().await.expect("warmup"); + + let source = harness.test_node(PAID_REPAIR_SOURCE_INDEX).expect("source"); + let target = harness.test_node(PAID_REPAIR_TARGET_INDEX).expect("target"); + let source_p2p = source.p2p_node.as_ref().expect("source p2p"); + let target_p2p = target.p2p_node.as_ref().expect("target p2p"); + let source_peer = *source_p2p.peer_id(); + let target_peer = *target_p2p.peer_id(); + + let content = b"paid-list-majority-authorizes-missing-replica-repair"; + let address = compute_address(content); + + assert!( + target_p2p + .dht_manager() + .is_in_routing_table(&source_peer) + .await, + "precondition: target must accept inbound hints from source in LocalRT" + ); + let storage_admission_peers: HashSet = target_p2p + .dht_manager() + .find_closest_nodes_local_with_self( + &address, + storage_admission_width(PAID_REPAIR_GROUP_SIZE), + ) + .await + .iter() + .map(|node| node.peer_id) + .collect(); + assert!( + storage_admission_peers.contains(&target_peer), + "precondition: target must be storage-admitted for the hinted key" + ); + let paid_group = target_p2p + .dht_manager() + .find_closest_nodes_local_with_self(&address, PAID_REPAIR_GROUP_SIZE) + .await; + assert_eq!( + paid_group.len(), + PAID_REPAIR_GROUP_SIZE, + "precondition: deterministic paid-list majority needs a full five-peer group" + ); + assert!( + paid_group.iter().any(|node| node.peer_id == target_peer), + "precondition: target must be in the paid-list close group" + ); + + let source_protocol = source.ant_protocol.as_ref().expect("source protocol"); + source_protocol + .storage() + .put(&address, content) + .await + .expect("put source record"); + + for idx in 0..harness.node_count() { + if let Some(protocol) = harness + .test_node(idx) + .and_then(|node| node.ant_protocol.as_ref()) + { + protocol.payment_verifier().cache_insert(address); + } + } + + for idx in 0..PAID_REPAIR_CONFIRMING_NODES { + let engine = harness + .test_node(idx) + .and_then(|node| node.replication_engine.as_ref()) + .expect("paid-list confirming engine"); + engine + .paid_list() + .insert(&address) + .await + .expect("paid-list insert"); + } + + let mut seeded_storage_holders = 0usize; + for idx in 0..harness.node_count() { + if let Some(protocol) = harness + .test_node(idx) + .and_then(|node| node.ant_protocol.as_ref()) + { + if protocol.storage().exists(&address).expect("exists check") { + seeded_storage_holders += 1; + } + } + } + assert_eq!( + seeded_storage_holders, 1, + "precondition: only the source should hold the record before repair" + ); + assert!( + seeded_storage_holders < PAID_REPAIR_STORAGE_QUORUM, + "precondition: storage quorum must be impossible without paid-list authorization" + ); + + let target_protocol = target.ant_protocol.as_ref().expect("target protocol"); + let target_engine = target.replication_engine.as_ref().expect("target engine"); + assert!( + !target_protocol.storage().exists(&address).expect("exists"), + "precondition: target starts without the record" + ); + assert!( + !target_engine + .paid_list() + .contains(&address) + .expect("contains"), + "precondition: target starts without local paid-list authorization" + ); + + let request = NeighborSyncRequest { + replica_hints: vec![address], + paid_hints: vec![], + bootstrapping: false, + commitment: None, + }; + let response = send_replication_request( + source_p2p, + &target_peer, + ReplicationMessage { + request_id: PAID_REPAIR_HINT_REQUEST_ID, + body: ReplicationMessageBody::NeighborSyncRequest(request), + }, + PAID_REPAIR_HINT_REQUEST_TIMEOUT, + ) + .await; + match response.body { + ReplicationMessageBody::NeighborSyncResponse(_) => {} + other => panic!("expected NeighborSyncResponse, got: {other:?}"), + } + + let deadline = tokio::time::Instant::now() + PAID_REPAIR_SETTLE_TIMEOUT; + let mut learned_paid = false; + let mut repaired_record = false; + while tokio::time::Instant::now() < deadline { + learned_paid = target_engine + .paid_list() + .contains(&address) + .expect("contains"); + repaired_record = target_protocol.storage().exists(&address).expect("exists"); + if learned_paid && repaired_record { + break; + } + tokio::time::sleep(PROPAGATION_POLL_INTERVAL).await; + } + + assert!( + learned_paid, + "target should learn paid-list authorization from remote majority" + ); + assert!( + repaired_record, + "paid-list majority should authorize fetching the missing replica" + ); + let fetched = target_protocol + .storage() + .get(&address) + .await + .expect("read repaired record") + .expect("repaired record should be present"); + assert_eq!(fetched, content, "target should store the fetched bytes"); + + harness.teardown().await.expect("teardown"); +} + // ========================================================================= // Section 18, Scenario #43: Paid-list persistence across restart // ========================================================================= diff --git a/tests/poc_bootstrap_stall.rs b/tests/poc_bootstrap_stall.rs index 4c0cc8cb..040ca3c4 100644 --- a/tests/poc_bootstrap_stall.rs +++ b/tests/poc_bootstrap_stall.rs @@ -81,12 +81,14 @@ fn peer(b: u8) -> PeerId { } fn entry(sender: PeerId) -> VerificationEntry { + let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: sender, } } diff --git a/tests/poc_d1_bounded_queues.rs b/tests/poc_d1_bounded_queues.rs index 79465f08..2aa66b21 100644 --- a/tests/poc_d1_bounded_queues.rs +++ b/tests/poc_d1_bounded_queues.rs @@ -59,12 +59,14 @@ fn unique_xorname(i: u32) -> [u8; 32] { } fn entry_from(sender: PeerId) -> VerificationEntry { + let now = Instant::now(); VerificationEntry { state: VerificationState::PendingVerify, pipeline: HintPipeline::Replica, verified_sources: Vec::new(), tried_sources: HashSet::new(), - created_at: Instant::now(), + created_at: now, + next_verify_at: now, hint_sender: sender, } }