From e1d6765f0ececcbf5760a6905653b9bae28d8587 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 23:08:59 -0700 Subject: [PATCH 01/31] Docs: plan catalog publication evidence --- docs/formats/segment-store-v1/requirements.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 3798a33..b52a56a 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -56,6 +56,28 @@ retention, or garbage collection. +## Catalog implementation evidence + +Issue #16 owns catalog-generation admission, publication, and immutable reader +snapshots. The following cases are planned evidence, not current behavior. + + + +| ID | Planned requirement | Oracle | Planned evidence | Status | +| --- | --- | --- | --- | --- | +| `KEEP-CATALOG-001` | `CatalogGeneration` admits positive values and refuses overflow when deriving a successor | Checked scalar model | `tests/catalog_generation.rs` | Planned in #16 | +| `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes | Independent golden corpus | `tests/catalog.rs`, `tests/publication_head.rs` | Planned in #16 | +| `KEEP-CATALOG-003` | Catalog entries are sorted by logical identity and duplicate keys are refused independently of input order | Ordered reference map | `tests/catalog_ordering.rs` | Planned in #16 | +| `KEEP-CATALOG-004` | Every admitted catalog location equals a verified top-level record span in the exact named segment | Segment parser and golden artifacts | `tests/catalog_locations.rs` | Planned in #16 | +| `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Planned in #16 | +| `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Planned in #16 | +| `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Planned in #16 | +| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented file and directory synchronization order | Fault-recording filesystem port | `tests/catalog_publication.rs` | Planned in #16 | +| `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Planned in #16 | +| `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Planned in #16 | + + + ## Compatibility and migration The byte grammars, magic values, field widths and order, endianness, kinds, From 410055703e33f10088fff71f74b880ca842ed40d Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 23:11:05 -0700 Subject: [PATCH 02/31] Add checked catalog generations --- src/catalog/generation.rs | 47 +++++++++++++++++++++++++++++++++ src/catalog/generation_error.rs | 29 ++++++++++++++++++++ src/catalog/mod.rs | 10 +++++++ src/lib.rs | 2 ++ tests/catalog_generation.rs | 39 +++++++++++++++++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 src/catalog/generation.rs create mode 100644 src/catalog/generation_error.rs create mode 100644 src/catalog/mod.rs create mode 100644 tests/catalog_generation.rs diff --git a/src/catalog/generation.rs b/src/catalog/generation.rs new file mode 100644 index 0000000..4050571 --- /dev/null +++ b/src/catalog/generation.rs @@ -0,0 +1,47 @@ +//! Checked positive catalog generation. + +use std::num::NonZeroU64; + +use super::CatalogGenerationError; + +/// Positive, canonically ordered catalog-generation coordinate. +/// +/// Generation `1` is the first published generation. A successor is admitted +/// only through checked arithmetic; overflow is a typed refusal. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CatalogGeneration(NonZeroU64); + +impl CatalogGeneration { + /// Admits one positive generation. + /// + /// # Errors + /// + /// Returns [`CatalogGenerationError::Zero`] when `value` is zero. + pub const fn new(value: u64) -> Result { + match NonZeroU64::new(value) { + Some(value) => Ok(Self(value)), + None => Err(CatalogGenerationError::Zero), + } + } + + /// Returns the exact positive generation value. + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } + + /// Derives the next generation through checked addition. + /// + /// # Errors + /// + /// Returns [`CatalogGenerationError::Exhausted`] when this generation is + /// `u64::MAX`. + pub const fn successor(self) -> Result { + let current = self.get(); + let Some(next) = current.checked_add(1) else { + return Err(CatalogGenerationError::Exhausted { current }); + }; + Self::new(next) + } +} diff --git a/src/catalog/generation_error.rs b/src/catalog/generation_error.rs new file mode 100644 index 0000000..d4ab035 --- /dev/null +++ b/src/catalog/generation_error.rs @@ -0,0 +1,29 @@ +//! Typed catalog-generation admission and transition failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit or advance a catalog generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogGenerationError { + /// Generation zero is outside the version-1 protocol. + Zero, + /// The current generation has no representable successor. + Exhausted { + /// The exact generation that could not advance. + current: u64, + }, +} + +impl fmt::Display for CatalogGenerationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero => formatter.write_str("catalog generation must be positive"), + Self::Exhausted { current } => { + write!(formatter, "catalog generation {current} has no successor") + } + } + } +} + +impl Error for CatalogGenerationError {} diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs new file mode 100644 index 0000000..279965b --- /dev/null +++ b/src/catalog/mod.rs @@ -0,0 +1,10 @@ +//! Catalog-generation domain coordinates and transition laws. +//! +//! This module owns semantic catalog generations. It does not own catalog byte +//! encoding, physical paths, filesystem publication, recovery, or retention. + +mod generation; +mod generation_error; + +pub use generation::CatalogGeneration; +pub use generation_error::CatalogGenerationError; diff --git a/src/lib.rs b/src/lib.rs index ba6eff5..a813856 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ mod adapters; mod blob; +mod catalog; mod chunk; mod layout; mod profile; @@ -33,6 +34,7 @@ pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, ByteRange, ByteRangeError, }; +pub use catalog::{CatalogGeneration, CatalogGenerationError}; pub use chunk::{ ChunkHashError, ChunkId, ChunkLength, ChunkOffset, ChunkSpan, ChunkingError, FastCdc, }; diff --git a/tests/catalog_generation.rs b/tests/catalog_generation.rs new file mode 100644 index 0000000..ad51216 --- /dev/null +++ b/tests/catalog_generation.rs @@ -0,0 +1,39 @@ +//! Public catalog-generation admission and transition laws. + +use keep::{CatalogGeneration, CatalogGenerationError}; + +#[test] +fn catalog_generation_admits_only_positive_values() -> Result<(), CatalogGenerationError> { + assert!(matches!( + CatalogGeneration::new(0), + Err(CatalogGenerationError::Zero) + )); + + let first = CatalogGeneration::new(1)?; + assert_eq!(first.get(), 1); + Ok(()) +} + +#[test] +fn catalog_generation_successor_is_checked() -> Result<(), CatalogGenerationError> { + let first = CatalogGeneration::new(1)?; + let second = first.successor()?; + assert_eq!(second.get(), 2); + + let maximum = CatalogGeneration::new(u64::MAX)?; + assert!(matches!( + maximum.successor(), + Err(CatalogGenerationError::Exhausted { current: u64::MAX }) + )); + Ok(()) +} + +#[test] +fn catalog_generation_order_is_canonical() -> Result<(), CatalogGenerationError> { + let first = CatalogGeneration::new(1)?; + let second = CatalogGeneration::new(2)?; + + assert!(first < second); + assert_eq!([second, first].into_iter().min(), Some(first)); + Ok(()) +} From 3fd39f8dd1f52c81b8dab18c4d341ed69d2af1ab Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 23:24:25 -0700 Subject: [PATCH 03/31] Add checksummed publication heads --- src/adapters/checksummed_publication_head.rs | 67 ++++++ src/adapters/framed_blake3.rs | 18 ++ src/adapters/mod.rs | 7 + src/adapters/publication_head_decode_error.rs | 79 +++++++ .../publication_head_decode_error_display.rs | 56 +++++ src/adapters/publication_head_decoder.rs | 174 +++++++++++++++ src/adapters/segment_seal_hash.rs | 20 +- src/catalog/digest.rs | 21 ++ src/catalog/length.rs | 49 ++++ src/catalog/length_error.rs | 43 ++++ src/catalog/mod.rs | 11 +- src/lib.rs | 13 +- tests/publication_head.rs | 210 ++++++++++++++++++ 13 files changed, 744 insertions(+), 24 deletions(-) create mode 100644 src/adapters/checksummed_publication_head.rs create mode 100644 src/adapters/framed_blake3.rs create mode 100644 src/adapters/publication_head_decode_error.rs create mode 100644 src/adapters/publication_head_decode_error_display.rs create mode 100644 src/adapters/publication_head_decoder.rs create mode 100644 src/catalog/digest.rs create mode 100644 src/catalog/length.rs create mode 100644 src/catalog/length_error.rs create mode 100644 tests/publication_head.rs diff --git a/src/adapters/checksummed_publication_head.rs b/src/adapters/checksummed_publication_head.rs new file mode 100644 index 0000000..87f8600 --- /dev/null +++ b/src/adapters/checksummed_publication_head.rs @@ -0,0 +1,67 @@ +//! Framing- and checksum-verified borrowed publication head. + +use super::{PublicationHeadDecodeError, publication_head_decoder}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Borrowed publication-head bytes with canonical framing and checksum proof. +/// +/// This state does not prove that the named catalog exists or that any catalog +/// entry names an admitted segment record. A reader must not treat it as a +/// complete catalog snapshot. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ChecksummedPublicationHead<'a> { + encoded: &'a [u8], + generation: CatalogGeneration, + catalog_length: CatalogLength, + catalog_digest: CatalogDigest, +} + +impl<'a> ChecksummedPublicationHead<'a> { + /// Decodes exact version-1 framing and verifies the head checksum. + /// + /// This operation performs no allocation or I/O. + /// + /// # Errors + /// + /// Returns [`PublicationHeadDecodeError`] for wrong framing, unsupported or + /// noncanonical fields, invalid coordinates, or checksum disagreement. + pub fn decode(encoded: &'a [u8]) -> Result { + publication_head_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(self) -> &'a [u8] { + self.encoded + } + + /// Returns the positive generation named by the head. + pub const fn generation(self) -> CatalogGeneration { + self.generation + } + + /// Returns the canonical length of the named catalog. + pub const fn catalog_length(self) -> CatalogLength { + self.catalog_length + } + + /// Returns the physical digest of the named catalog. + pub const fn catalog_digest(self) -> CatalogDigest { + self.catalog_digest + } + + pub(super) const fn from_verified_parts( + encoded: &'a [u8], + generation: CatalogGeneration, + catalog_length: CatalogLength, + catalog_digest: CatalogDigest, + ) -> Self { + Self { + encoded, + generation, + catalog_length, + catalog_digest, + } + } +} diff --git a/src/adapters/framed_blake3.rs b/src/adapters/framed_blake3.rs new file mode 100644 index 0000000..f40d7ba --- /dev/null +++ b/src/adapters/framed_blake3.rs @@ -0,0 +1,18 @@ +//! Named version-1 framed BLAKE3 boundary primitive. + +use blake3::Hasher; + +const VERSION: u16 = 1; +const ALGORITHM: u8 = 1; + +pub(super) fn hash(domain: &[u8], parts: &[&[u8]], length: u64) -> [u8; 32] { + let mut hasher = Hasher::new(); + hasher.update(domain); + hasher.update(&VERSION.to_be_bytes()); + hasher.update(&[ALGORITHM]); + for part in parts { + hasher.update(part); + } + hasher.update(&length.to_be_bytes()); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 1ac3a94..1b0d38e 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -11,8 +11,10 @@ mod blob_id_binary; mod blob_id_binary_error; mod blob_id_text; mod blob_id_text_error; +mod checksummed_publication_head; mod checksummed_segment_record; mod filesystem_segment_stage; +mod framed_blake3; mod layout_decode_error; mod layout_decode_error_display; mod layout_decode_policy; @@ -28,6 +30,9 @@ mod layout_record_encoder; mod layout_record_format; mod layout_record_framing; mod lower_hex; +mod publication_head_decode_error; +mod publication_head_decode_error_display; +mod publication_head_decoder; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -89,6 +94,7 @@ pub use admitted_segment::AdmittedSegment; pub use admitted_segment_record::AdmittedSegmentRecord; pub use blob_id_binary_error::BlobIdBinaryParseError; pub use blob_id_text_error::BlobIdTextParseError; +pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; pub use filesystem_segment_stage::FilesystemSegmentStage; pub use layout_decode_error::LayoutDecodeError; @@ -97,6 +103,7 @@ pub use layout_encode_error::LayoutEncodeError; pub use layout_id_binary_error::LayoutIdBinaryParseError; pub use layout_id_text_error::LayoutIdTextParseError; pub use layout_record::CanonicalLayoutRecord; +pub use publication_head_decode_error::PublicationHeadDecodeError; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/publication_head_decode_error.rs b/src/adapters/publication_head_decode_error.rs new file mode 100644 index 0000000..dfb6fe0 --- /dev/null +++ b/src/adapters/publication_head_decode_error.rs @@ -0,0 +1,79 @@ +//! Publication-head decoding failures. + +use crate::{CatalogGenerationError, CatalogLengthError}; + +/// Failure to decode and checksum-verify a version-1 publication head. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PublicationHeadDecodeError { + /// The input is not exactly one complete head. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed format magic did not match. + InvalidMagic { + /// Bounded magic observed in the input. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Version implemented by this decoder. + expected: u16, + /// Version declared by the input. + observed: u16, + }, + /// Version-1 flags were nonzero. + Flags { + /// Canonical flag field. + expected: u16, + /// Observed flag field. + observed: u16, + }, + /// The fixed head length field was noncanonical. + HeadLength { + /// Canonical length field. + expected: u16, + /// Observed length field. + observed: u16, + }, + /// The head checksum algorithm is unsupported. + ChecksumAlgorithm { + /// Algorithm implemented by this decoder. + expected: u8, + /// Algorithm declared by the input. + observed: u8, + }, + /// The catalog digest algorithm is unsupported. + DigestAlgorithm { + /// Algorithm implemented by this decoder. + expected: u8, + /// Algorithm declared by the input. + observed: u8, + }, + /// The generation coordinate was invalid. + Generation { + /// Exact generation admission failure. + source: CatalogGenerationError, + }, + /// The catalog length coordinate was invalid. + CatalogLength { + /// Exact catalog-length admission failure. + source: CatalogLengthError, + }, + /// Version-1 reserved bytes were nonzero. + Reserved { + /// Required all-zero bytes. + expected: [u8; 24], + /// Observed reserved bytes. + observed: [u8; 24], + }, + /// The stored checksum disagreed with the canonical framed hash. + ChecksumMismatch { + /// Checksum derived from the covered bytes. + expected: [u8; 32], + /// Checksum stored in the head. + observed: [u8; 32], + }, +} diff --git a/src/adapters/publication_head_decode_error_display.rs b/src/adapters/publication_head_decode_error_display.rs new file mode 100644 index 0000000..3fff390 --- /dev/null +++ b/src/adapters/publication_head_decode_error_display.rs @@ -0,0 +1,56 @@ +//! Human-readable publication-head decoding diagnostics. + +use std::error::Error; +use std::fmt; + +use super::PublicationHeadDecodeError; + +impl fmt::Display for PublicationHeadDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "wrong publication-head length: expected {expected}, observed {observed}" + ), + Self::InvalidMagic { .. } => formatter.write_str("invalid publication-head magic"), + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported publication-head version {observed}; version {expected} is required" + ), + Self::Flags { expected, observed } => write!( + formatter, + "noncanonical publication-head flags: expected {expected}, observed {observed}" + ), + Self::HeadLength { expected, observed } => write!( + formatter, + "wrong publication-head length field: expected {expected}, observed {observed}" + ), + Self::ChecksumAlgorithm { expected, observed } => write!( + formatter, + "unsupported head checksum algorithm {observed}; algorithm {expected} is required" + ), + Self::DigestAlgorithm { expected, observed } => write!( + formatter, + "unsupported catalog digest algorithm {observed}; algorithm {expected} is required" + ), + Self::Generation { source } => write!(formatter, "invalid head generation: {source}"), + Self::CatalogLength { source } => { + write!(formatter, "invalid head catalog length: {source}") + } + Self::Reserved { .. } => formatter.write_str("nonzero publication-head reserved bytes"), + Self::ChecksumMismatch { .. } => { + formatter.write_str("publication-head checksum mismatch") + } + } + } +} + +impl Error for PublicationHeadDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Generation { source } => Some(source), + Self::CatalogLength { source } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/publication_head_decoder.rs b/src/adapters/publication_head_decoder.rs new file mode 100644 index 0000000..ac1bd7c --- /dev/null +++ b/src/adapters/publication_head_decoder.rs @@ -0,0 +1,174 @@ +//! Canonical publication-head framing and checksum decoder. + +use super::{ChecksummedPublicationHead, PublicationHeadDecodeError, framed_blake3}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +const ENCODED_LENGTH: usize = 128; +const MAGIC: [u8; 16] = *b"KEEP:CATHEAD:V1\0"; +const VERSION: u16 = 1; +const FLAGS: u16 = 0; +const HEAD_LENGTH: u16 = 128; +const ALGORITHM: u8 = 1; +const CHECKSUM_INPUT_LENGTH: usize = 96; +const CHECKSUM_DOMAIN: &[u8] = b"KEEP:CATHEAD:SUM\0"; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, PublicationHeadDecodeError> { + if encoded.len() != ENCODED_LENGTH { + return Err(PublicationHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }); + } + let fields = decode_fields(encoded)?; + let (generation, catalog_length) = validate_fields(&fields)?; + validate_checksum(encoded, fields.checksum)?; + Ok(ChecksummedPublicationHead::from_verified_parts( + encoded, + generation, + catalog_length, + CatalogDigest::from_validated(fields.catalog_digest), + )) +} + +fn validate_fields( + fields: &DecodedFields, +) -> Result<(CatalogGeneration, CatalogLength), PublicationHeadDecodeError> { + require_eq(fields.magic, MAGIC, |observed| { + PublicationHeadDecodeError::InvalidMagic { observed } + })?; + require_eq(fields.version, VERSION, |observed| { + PublicationHeadDecodeError::UnsupportedVersion { + expected: VERSION, + observed, + } + })?; + require_eq(fields.flags, FLAGS, |observed| { + PublicationHeadDecodeError::Flags { + expected: FLAGS, + observed, + } + })?; + require_eq(fields.head_length, HEAD_LENGTH, |observed| { + PublicationHeadDecodeError::HeadLength { + expected: HEAD_LENGTH, + observed, + } + })?; + require_eq(fields.checksum_algorithm, ALGORITHM, |observed| { + PublicationHeadDecodeError::ChecksumAlgorithm { + expected: ALGORITHM, + observed, + } + })?; + require_eq(fields.digest_algorithm, ALGORITHM, |observed| { + PublicationHeadDecodeError::DigestAlgorithm { + expected: ALGORITHM, + observed, + } + })?; + let generation = CatalogGeneration::new(fields.generation) + .map_err(|source| PublicationHeadDecodeError::Generation { source })?; + let catalog_length = CatalogLength::new(fields.catalog_length) + .map_err(|source| PublicationHeadDecodeError::CatalogLength { source })?; + let expected = [0_u8; 24]; + require_eq(fields.reserved, expected, |observed| { + PublicationHeadDecodeError::Reserved { expected, observed } + })?; + Ok((generation, catalog_length)) +} + +fn validate_checksum(encoded: &[u8], observed: [u8; 32]) -> Result<(), PublicationHeadDecodeError> { + let covered = + encoded + .get(..CHECKSUM_INPUT_LENGTH) + .ok_or(PublicationHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + })?; + let expected = framed_blake3::hash( + CHECKSUM_DOMAIN, + &[covered], + u64::try_from(CHECKSUM_INPUT_LENGTH).map_err(|_source| { + PublicationHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + } + })?, + ); + require_eq(observed, expected, |observed| { + PublicationHeadDecodeError::ChecksumMismatch { expected, observed } + }) +} + +fn decode_fields(encoded: &[u8]) -> Result { + Ok(DecodedFields { + magic: read_array(encoded, 0)?, + version: u16::from_be_bytes(read_array(encoded, 16)?), + flags: u16::from_be_bytes(read_array(encoded, 18)?), + head_length: u16::from_be_bytes(read_array(encoded, 20)?), + checksum_algorithm: read_u8(encoded, 22)?, + digest_algorithm: read_u8(encoded, 23)?, + generation: u64::from_be_bytes(read_array(encoded, 24)?), + catalog_length: u64::from_be_bytes(read_array(encoded, 32)?), + catalog_digest: read_array(encoded, 40)?, + reserved: read_array(encoded, 72)?, + checksum: read_array(encoded, 96)?, + }) +} + +fn read_u8(encoded: &[u8], offset: usize) -> Result { + encoded + .get(offset) + .copied() + .ok_or(PublicationHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; LENGTH], PublicationHeadDecodeError> { + let end = offset + .checked_add(LENGTH) + .ok_or(PublicationHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + })?; + encoded + .get(offset..end) + .and_then(|field| field.try_into().ok()) + .ok_or(PublicationHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }) +} + +fn require_eq( + observed: T, + expected: T, + error: impl FnOnce(T) -> PublicationHeadDecodeError, +) -> Result<(), PublicationHeadDecodeError> { + if observed == expected { + Ok(()) + } else { + Err(error(observed)) + } +} + +struct DecodedFields { + magic: [u8; 16], + version: u16, + flags: u16, + head_length: u16, + checksum_algorithm: u8, + digest_algorithm: u8, + generation: u64, + catalog_length: u64, + catalog_digest: [u8; 32], + reserved: [u8; 24], + checksum: [u8; 32], +} diff --git a/src/adapters/segment_seal_hash.rs b/src/adapters/segment_seal_hash.rs index 9fece04..481767e 100644 --- a/src/adapters/segment_seal_hash.rs +++ b/src/adapters/segment_seal_hash.rs @@ -1,9 +1,7 @@ //! Canonical physical segment digest and seal checksum calculation. -use blake3::Hasher; - use super::segment_seal::ENCODED_LENGTH; -use super::{SegmentDigest, SegmentSealError}; +use super::{SegmentDigest, SegmentSealError, framed_blake3}; pub(super) const VERSION: u16 = 1; pub(super) const ALGORITHM: u8 = 1; @@ -29,7 +27,7 @@ pub(super) fn segment_digest( .ok_or(SegmentSealError::DigestLengthArithmetic { prefix_length: prefix.len(), })?; - Ok(SegmentDigest::from_validated(hash_parts( + Ok(SegmentDigest::from_validated(framed_blake3::hash( DIGEST_DOMAIN, &[prefix, q], input_length, @@ -43,21 +41,9 @@ pub(super) fn seal_checksum(seal: &[u8]) -> Result<[u8; 32], SegmentSealError> { observed: seal.len(), }, )?; - Ok(hash_parts( + Ok(framed_blake3::hash( CHECKSUM_DOMAIN, &[covered], u64::from(SEAL_CHECKSUM_INPUT_LENGTH), )) } - -fn hash_parts(domain: &[u8], parts: &[&[u8]], length: u64) -> [u8; 32] { - let mut hasher = Hasher::new(); - hasher.update(domain); - hasher.update(&VERSION.to_be_bytes()); - hasher.update(&[ALGORITHM]); - for part in parts { - hasher.update(part); - } - hasher.update(&length.to_be_bytes()); - *hasher.finalize().as_bytes() -} diff --git a/src/catalog/digest.rs b/src/catalog/digest.rs new file mode 100644 index 0000000..cf1dc7f --- /dev/null +++ b/src/catalog/digest.rs @@ -0,0 +1,21 @@ +//! Verified physical catalog-generation digest. + +/// Verified digest of one exact immutable catalog generation. +/// +/// This value is a physical generation coordinate and predecessor witness. It +/// is not a logical content identity and does not establish retention. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CatalogDigest([u8; 32]); + +impl CatalogDigest { + /// Returns the exact canonical digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(crate) const fn from_validated(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/catalog/length.rs b/src/catalog/length.rs new file mode 100644 index 0000000..8e88a6e --- /dev/null +++ b/src/catalog/length.rs @@ -0,0 +1,49 @@ +//! Checked canonical catalog byte length. + +use super::CatalogLengthError; + +const HEADER_LENGTH: u64 = 128; +const ENTRY_LENGTH: u64 = 160; +const TRAILER_LENGTH: u64 = 64; +const MINIMUM: u64 = HEADER_LENGTH + TRAILER_LENGTH; +const MAXIMUM: u64 = 167_772_352; + +/// Exact canonical byte length of one complete version-1 catalog. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CatalogLength(u64); + +impl CatalogLength { + /// Admits a complete version-1 catalog length. + /// + /// # Errors + /// + /// Returns [`CatalogLengthError`] when `value` exceeds the format bound or + /// cannot contain a whole number of fixed-width entries. + pub const fn new(value: u64) -> Result { + if value < MINIMUM || value > MAXIMUM { + return Err(CatalogLengthError::OutOfBounds { + minimum: MINIMUM, + maximum: MAXIMUM, + observed: value, + }); + } + let Some(entry_bytes) = value.checked_sub(MINIMUM) else { + return Err(CatalogLengthError::OutOfBounds { + minimum: MINIMUM, + maximum: MAXIMUM, + observed: value, + }); + }; + if !entry_bytes.is_multiple_of(ENTRY_LENGTH) { + return Err(CatalogLengthError::NotCongruent { observed: value }); + } + Ok(Self(value)) + } + + /// Returns the exact admitted byte length. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} diff --git a/src/catalog/length_error.rs b/src/catalog/length_error.rs new file mode 100644 index 0000000..f3200e9 --- /dev/null +++ b/src/catalog/length_error.rs @@ -0,0 +1,43 @@ +//! Catalog-length admission failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit a canonical catalog byte length. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogLengthError { + /// The length is outside the version-1 catalog bounds. + OutOfBounds { + /// Smallest complete catalog length. + minimum: u64, + /// Largest permitted catalog length. + maximum: u64, + /// Length supplied by the boundary. + observed: u64, + }, + /// The length cannot contain a whole number of fixed-width entries. + NotCongruent { + /// Length supplied by the boundary. + observed: u64, + }, +} + +impl fmt::Display for CatalogLengthError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::OutOfBounds { + minimum, + maximum, + observed, + } => write!( + formatter, + "catalog length {observed} is outside {minimum}..={maximum}" + ), + Self::NotCongruent { observed } => { + write!(formatter, "catalog length {observed} is not congruent") + } + } + } +} + +impl Error for CatalogLengthError {} diff --git a/src/catalog/mod.rs b/src/catalog/mod.rs index 279965b..394dc75 100644 --- a/src/catalog/mod.rs +++ b/src/catalog/mod.rs @@ -1,10 +1,17 @@ //! Catalog-generation domain coordinates and transition laws. //! -//! This module owns semantic catalog generations. It does not own catalog byte -//! encoding, physical paths, filesystem publication, recovery, or retention. +//! This module owns semantic catalog generations and checked physical catalog +//! coordinates. It does not own catalog byte encoding, physical paths, +//! filesystem publication, recovery, or retention. +mod digest; mod generation; mod generation_error; +mod length; +mod length_error; +pub use digest::CatalogDigest; pub use generation::CatalogGeneration; pub use generation_error::CatalogGenerationError; +pub use length::CatalogLength; +pub use length_error::CatalogLengthError; diff --git a/src/lib.rs b/src/lib.rs index a813856..82b708f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,10 +21,11 @@ mod reference; pub use adapters::{ AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalLayoutRecord, ChecksummedSegmentRecord, FilesystemSegmentStage, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, - SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + CanonicalLayoutRecord, ChecksummedPublicationHead, ChecksummedSegmentRecord, + FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, @@ -34,7 +35,9 @@ pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, ByteRange, ByteRangeError, }; -pub use catalog::{CatalogGeneration, CatalogGenerationError}; +pub use catalog::{ + CatalogDigest, CatalogGeneration, CatalogGenerationError, CatalogLength, CatalogLengthError, +}; pub use chunk::{ ChunkHashError, ChunkId, ChunkLength, ChunkOffset, ChunkSpan, ChunkingError, FastCdc, }; diff --git a/tests/publication_head.rs b/tests/publication_head.rs new file mode 100644 index 0000000..0b6ec03 --- /dev/null +++ b/tests/publication_head.rs @@ -0,0 +1,210 @@ +//! Public publication-head framing and checksum laws. + +mod support; + +use std::error::Error; + +use keep::{ + CatalogGenerationError, CatalogLengthError, ChecksummedPublicationHead, + PublicationHeadDecodeError, +}; +use support::{decode_hex, require_error}; + +const GENERATION_ONE_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const GENERATION_TWO_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); +const BUNDLE_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const GENERATION_ONE_DIGEST_HEX: &str = + "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320"; +const GENERATION_TWO_DIGEST_HEX: &str = + "ea7d0055fd21f00ed94809ef4e671d72fa2e6a4a5d9ecefb23f3a320a2dad993"; +const BUNDLE_DIGEST_HEX: &str = "0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811"; +const MAGIC_OFFSET: usize = 0; +const VERSION_OFFSET: usize = 16; +const FLAGS_OFFSET: usize = 18; +const HEAD_LENGTH_OFFSET: usize = 20; +const CHECKSUM_ALGORITHM_OFFSET: usize = 22; +const DIGEST_ALGORITHM_OFFSET: usize = 23; +const GENERATION_OFFSET: usize = 24; +const CATALOG_LENGTH_OFFSET: usize = 32; +const RESERVED_OFFSET: usize = 72; +const CHECKSUM_OFFSET: usize = 96; + +#[test] +fn frozen_publication_heads_are_checksum_verified_exactly() -> Result<(), Box> { + let generation_one = head_bytes(GENERATION_ONE_HEX)?; + let first = ChecksummedPublicationHead::decode(&generation_one)?; + assert_eq!(first.generation().get(), 1); + assert_eq!(first.catalog_length().get(), 352); + assert_eq!( + first.catalog_digest().as_bytes().as_slice(), + decode_hex(GENERATION_ONE_DIGEST_HEX)? + ); + assert_eq!(first.encoded(), generation_one); + + let generation_two = head_bytes(GENERATION_TWO_HEX)?; + let second = ChecksummedPublicationHead::decode(&generation_two)?; + assert_eq!(second.generation().get(), 2); + assert_eq!(second.catalog_length().get(), 352); + assert_eq!( + second.catalog_digest().as_bytes().as_slice(), + decode_hex(GENERATION_TWO_DIGEST_HEX)? + ); + + let bundle = head_bytes(BUNDLE_HEX)?; + let bundle_head = ChecksummedPublicationHead::decode(&bundle)?; + assert_eq!(bundle_head.generation().get(), 1); + assert_eq!(bundle_head.catalog_length().get(), 512); + assert_eq!( + bundle_head.catalog_digest().as_bytes().as_slice(), + decode_hex(BUNDLE_DIGEST_HEX)? + ); + Ok(()) +} + +#[test] +fn publication_head_refuses_noncanonical_fixed_fields() -> Result<(), Box> { + assert_refusal(MAGIC_OFFSET, 0, |error| { + matches!(error, PublicationHeadDecodeError::InvalidMagic { .. }) + })?; + assert_refusal(VERSION_OFFSET + 1, 2, |error| { + error + == PublicationHeadDecodeError::UnsupportedVersion { + expected: 1, + observed: 2, + } + })?; + assert_refusal(FLAGS_OFFSET + 1, 1, |error| { + error + == PublicationHeadDecodeError::Flags { + expected: 0, + observed: 1, + } + })?; + assert_refusal(HEAD_LENGTH_OFFSET + 1, 127, |error| { + error + == PublicationHeadDecodeError::HeadLength { + expected: 128, + observed: 127, + } + })?; + assert_refusal(CHECKSUM_ALGORITHM_OFFSET, 2, |error| { + error + == PublicationHeadDecodeError::ChecksumAlgorithm { + expected: 1, + observed: 2, + } + })?; + assert_refusal(DIGEST_ALGORITHM_OFFSET, 2, |error| { + error + == PublicationHeadDecodeError::DigestAlgorithm { + expected: 1, + observed: 2, + } + })?; + assert_refusal(RESERVED_OFFSET, 1, |error| { + matches!(error, PublicationHeadDecodeError::Reserved { .. }) + })?; + Ok(()) +} + +#[test] +fn publication_head_refuses_invalid_generation_and_catalog_length() -> Result<(), Box> { + assert_refusal(GENERATION_OFFSET + 7, 0, |error| { + error + == PublicationHeadDecodeError::Generation { + source: CatalogGenerationError::Zero, + } + })?; + assert_u64_refusal(CATALOG_LENGTH_OFFSET, 191, |error| { + error + == PublicationHeadDecodeError::CatalogLength { + source: CatalogLengthError::OutOfBounds { + minimum: 192, + maximum: 167_772_352, + observed: 191, + }, + } + })?; + assert_u64_refusal(CATALOG_LENGTH_OFFSET, 193, |error| { + error + == PublicationHeadDecodeError::CatalogLength { + source: CatalogLengthError::NotCongruent { observed: 193 }, + } + })?; + Ok(()) +} + +#[test] +fn publication_head_refuses_wrong_width_and_checksum() -> Result<(), Box> { + let mut encoded = head_bytes(GENERATION_ONE_HEX)?; + let _last = encoded.pop().ok_or("head fixture is empty")?; + assert_eq!( + require_error( + ChecksummedPublicationHead::decode(&encoded), + "truncated head was admitted" + )?, + PublicationHeadDecodeError::WrongLength { + expected: 128, + observed: 127, + } + ); + + let mut corrupt = head_bytes(GENERATION_ONE_HEX)?; + let checksum_byte = corrupt + .get_mut(CHECKSUM_OFFSET) + .ok_or("head fixture lacks its checksum")?; + *checksum_byte ^= 1; + assert!(matches!( + require_error( + ChecksummedPublicationHead::decode(&corrupt), + "corrupt head checksum was admitted" + )?, + PublicationHeadDecodeError::ChecksumMismatch { .. } + )); + Ok(()) +} + +fn assert_refusal( + offset: usize, + value: u8, + predicate: impl FnOnce(PublicationHeadDecodeError) -> bool, +) -> Result<(), Box> { + let mut encoded = head_bytes(GENERATION_ONE_HEX)?; + let field = encoded + .get_mut(offset) + .ok_or("head fixture lacks the mutation offset")?; + *field = value; + let error = require_error( + ChecksummedPublicationHead::decode(&encoded), + "mutated head was admitted", + )?; + assert!(predicate(error), "unexpected refusal: {error:?}"); + Ok(()) +} + +fn assert_u64_refusal( + offset: usize, + value: u64, + predicate: impl FnOnce(PublicationHeadDecodeError) -> bool, +) -> Result<(), Box> { + let mut encoded = head_bytes(GENERATION_ONE_HEX)?; + let field = encoded + .get_mut(offset..offset.checked_add(8).ok_or("test offset overflow")?) + .ok_or("head fixture lacks the u64 mutation field")?; + field.copy_from_slice(&value.to_be_bytes()); + let error = require_error( + ChecksummedPublicationHead::decode(&encoded), + "mutated head was admitted", + )?; + assert!(predicate(error), "unexpected refusal: {error:?}"); + Ok(()) +} + +fn head_bytes(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("head fixture must end in one LF")?, + ) + .map_err(Into::into) +} From ac06c4089e5c6c3cbac485b4916292d3fcf124f4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 23:42:51 -0700 Subject: [PATCH 04/31] Add checksummed catalogs --- src/adapters/catalog_decode_error.rs | 160 ++++++++++++++++ src/adapters/catalog_decode_error_display.rs | 151 +++++++++++++++ src/adapters/catalog_decoder.rs | 173 ++++++++++++++++++ src/adapters/catalog_entry_decode_error.rs | 111 +++++++++++ .../catalog_entry_decode_error_display.rs | 101 ++++++++++ src/adapters/catalog_entry_decoder.rs | 173 ++++++++++++++++++ src/adapters/catalog_entry_fields.rs | 40 ++++ src/adapters/catalog_entry_sequence.rs | 57 ++++++ src/adapters/catalog_header_decoder.rs | 76 ++++++++ src/adapters/catalog_integrity.rs | 71 +++++++ src/adapters/checksummed_catalog.rs | 77 ++++++++ src/adapters/mod.rs | 14 ++ src/adapters/segment_record_checksum.rs | 19 +- src/lib.rs | 19 +- tests/catalog.rs | 93 ++++++++++ tests/catalog/entry_laws.rs | 85 +++++++++ tests/catalog/format_oracle.rs | 58 ++++++ tests/catalog/header_laws.rs | 117 ++++++++++++ tests/catalog/integrity_laws.rs | 51 ++++++ tests/catalog/mutation_support.rs | 87 +++++++++ tests/catalog/ordering_laws.rs | 52 ++++++ 21 files changed, 1763 insertions(+), 22 deletions(-) create mode 100644 src/adapters/catalog_decode_error.rs create mode 100644 src/adapters/catalog_decode_error_display.rs create mode 100644 src/adapters/catalog_decoder.rs create mode 100644 src/adapters/catalog_entry_decode_error.rs create mode 100644 src/adapters/catalog_entry_decode_error_display.rs create mode 100644 src/adapters/catalog_entry_decoder.rs create mode 100644 src/adapters/catalog_entry_fields.rs create mode 100644 src/adapters/catalog_entry_sequence.rs create mode 100644 src/adapters/catalog_header_decoder.rs create mode 100644 src/adapters/catalog_integrity.rs create mode 100644 src/adapters/checksummed_catalog.rs create mode 100644 tests/catalog.rs create mode 100644 tests/catalog/entry_laws.rs create mode 100644 tests/catalog/format_oracle.rs create mode 100644 tests/catalog/header_laws.rs create mode 100644 tests/catalog/integrity_laws.rs create mode 100644 tests/catalog/mutation_support.rs create mode 100644 tests/catalog/ordering_laws.rs diff --git a/src/adapters/catalog_decode_error.rs b/src/adapters/catalog_decode_error.rs new file mode 100644 index 0000000..df32aff --- /dev/null +++ b/src/adapters/catalog_decode_error.rs @@ -0,0 +1,160 @@ +//! Catalog decoding failures. + +use super::CatalogEntryDecodeError; +use crate::{CatalogGenerationError, CatalogLengthError}; + +/// Failure to decode and integrity-verify one version-1 catalog. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogDecodeError { + /// The input is too short to contain fixed catalog framing. + MinimumLength { + /// Smallest complete catalog. + minimum: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed format magic did not match. + InvalidMagic { + /// Bounded magic observed in the input. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Version implemented by this decoder. + expected: u16, + /// Version declared by the input. + observed: u16, + }, + /// Version-1 flags were nonzero. + Flags { + /// Canonical flags. + expected: u16, + /// Observed flags. + observed: u16, + }, + /// The fixed header-width field was noncanonical. + HeaderLength { + /// Canonical header width. + expected: u16, + /// Observed header width. + observed: u16, + }, + /// The fixed entry-width field was noncanonical. + EntryLength { + /// Canonical entry width. + expected: u16, + /// Observed entry width. + observed: u16, + }, + /// The generation coordinate was invalid. + Generation { + /// Exact generation admission failure. + source: CatalogGenerationError, + }, + /// Generation 1 carried a forbidden predecessor digest. + UnexpectedPredecessor { + /// Observed generation. + generation: u64, + /// Observed nonzero predecessor. + observed: [u8; 32], + }, + /// A later generation omitted its required predecessor digest. + MissingPredecessor { + /// Observed generation. + generation: u64, + }, + /// The declared entry count exceeded the format bound. + EntryCountOutOfBounds { + /// Largest supported entry count. + maximum: u64, + /// Declared entry count. + observed: u64, + }, + /// The declared catalog length was invalid. + CatalogLength { + /// Exact catalog-length admission failure. + source: CatalogLengthError, + }, + /// Entry count and declared length disagreed. + EntryCountLengthMismatch { + /// Declared entry count. + entry_count: u64, + /// Length derived by checked format arithmetic. + expected: u64, + /// Declared catalog length. + observed: u64, + }, + /// The actual input width disagreed with the declared canonical width. + ObservedLength { + /// Declared canonical width. + declared: u64, + /// Actual input width. + observed: usize, + }, + /// Catalog-length arithmetic overflowed. + LengthArithmetic { + /// Entry count participating in the calculation. + entry_count: u64, + }, + /// A host byte width could not enter the canonical hash frame. + HashLength { + /// Host byte width supplied to the frame. + observed: usize, + }, + /// The catalog checksum algorithm is unsupported. + ChecksumAlgorithm { + /// Algorithm implemented by this decoder. + expected: u8, + /// Algorithm declared by the input. + observed: u8, + }, + /// The catalog digest algorithm is unsupported. + DigestAlgorithm { + /// Algorithm implemented by this decoder. + expected: u8, + /// Algorithm declared by the input. + observed: u8, + }, + /// Version-1 header reserved bytes were nonzero. + Reserved { + /// Required all-zero bytes. + expected: [u8; 46], + /// Observed reserved bytes. + observed: [u8; 46], + }, + /// One fixed-width catalog entry was invalid. + Entry { + /// Zero-based canonical entry index. + index: u64, + /// Exact entry admission failure. + source: CatalogEntryDecodeError, + }, + /// Two entries carried the same logical identity. + DuplicateIdentity { + /// First occurrence. + first_index: u64, + /// Duplicate occurrence. + duplicate_index: u64, + }, + /// Logical identities were not in canonical order. + IdentityOrder { + /// Earlier physical entry index. + previous_index: u64, + /// Out-of-order physical entry index. + observed_index: u64, + }, + /// The stored checksum disagreed with canonical catalog bytes. + ChecksumMismatch { + /// Derived checksum. + expected: [u8; 32], + /// Stored checksum. + observed: [u8; 32], + }, + /// The stored physical digest disagreed with canonical catalog bytes. + DigestMismatch { + /// Derived digest. + expected: [u8; 32], + /// Stored digest. + observed: [u8; 32], + }, +} diff --git a/src/adapters/catalog_decode_error_display.rs b/src/adapters/catalog_decode_error_display.rs new file mode 100644 index 0000000..6fbdb14 --- /dev/null +++ b/src/adapters/catalog_decode_error_display.rs @@ -0,0 +1,151 @@ +//! Human-readable catalog decoding diagnostics. + +use std::error::Error; +use std::fmt; + +use super::CatalogDecodeError; + +impl fmt::Display for CatalogDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MinimumLength { minimum, observed } => { + display_minimum_length(formatter, *minimum, *observed) + } + Self::InvalidMagic { .. } => formatter.write_str("invalid catalog magic"), + Self::UnsupportedVersion { expected, observed } => { + write!( + formatter, + "unsupported catalog version {observed}; version {expected} is required" + ) + } + Self::Flags { expected, observed } => { + write!( + formatter, + "noncanonical catalog flags: expected {expected}, observed {observed}" + ) + } + Self::HeaderLength { expected, observed } => { + write!( + formatter, + "wrong catalog header length: expected {expected}, observed {observed}" + ) + } + Self::EntryLength { expected, observed } => { + write!( + formatter, + "wrong catalog entry length: expected {expected}, observed {observed}" + ) + } + Self::Generation { source } => { + write!(formatter, "invalid catalog generation: {source}") + } + Self::UnexpectedPredecessor { generation, .. } => { + write!( + formatter, + "catalog generation {generation} forbids a predecessor" + ) + } + Self::MissingPredecessor { generation } => { + write!( + formatter, + "catalog generation {generation} requires a predecessor" + ) + } + Self::EntryCountOutOfBounds { maximum, observed } => { + write!( + formatter, + "catalog entry count {observed} exceeds {maximum}" + ) + } + Self::CatalogLength { source } => write!(formatter, "invalid catalog length: {source}"), + Self::EntryCountLengthMismatch { + entry_count, + expected, + observed, + } => write!( + formatter, + "catalog count {entry_count} requires length {expected}, observed {observed}" + ), + Self::ObservedLength { declared, observed } => { + write!( + formatter, + "catalog declares {declared} bytes, observed {observed}" + ) + } + Self::LengthArithmetic { entry_count } => { + write!( + formatter, + "catalog length arithmetic failed for {entry_count} entries" + ) + } + Self::HashLength { observed } => display_hash_length(formatter, *observed), + Self::ChecksumAlgorithm { expected, observed } => { + display_algorithm(formatter, "checksum", *expected, *observed) + } + Self::DigestAlgorithm { expected, observed } => { + display_algorithm(formatter, "digest", *expected, *observed) + } + Self::Reserved { .. } => formatter.write_str("nonzero catalog header reserved bytes"), + Self::Entry { index, source } => { + write!(formatter, "invalid catalog entry {index}: {source}") + } + Self::DuplicateIdentity { + first_index, + duplicate_index, + } => write!( + formatter, + "catalog identity at {duplicate_index} duplicates entry {first_index}" + ), + Self::IdentityOrder { + previous_index, + observed_index, + } => write!( + formatter, + "catalog identity at {observed_index} precedes entry {previous_index}" + ), + Self::ChecksumMismatch { .. } => formatter.write_str("catalog checksum mismatch"), + Self::DigestMismatch { .. } => formatter.write_str("catalog digest mismatch"), + } + } +} + +fn display_minimum_length( + formatter: &mut fmt::Formatter<'_>, + minimum: usize, + observed: usize, +) -> fmt::Result { + write!( + formatter, + "catalog requires at least {minimum} bytes, observed {observed}" + ) +} + +fn display_hash_length(formatter: &mut fmt::Formatter<'_>, observed: usize) -> fmt::Result { + write!( + formatter, + "catalog hash length cannot represent {observed} bytes" + ) +} + +fn display_algorithm( + formatter: &mut fmt::Formatter<'_>, + field: &str, + expected: u8, + observed: u8, +) -> fmt::Result { + write!( + formatter, + "unsupported catalog {field} algorithm {observed}; algorithm {expected} is required" + ) +} + +impl Error for CatalogDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Generation { source } => Some(source), + Self::CatalogLength { source } => Some(source), + Self::Entry { source, .. } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/catalog_decoder.rs b/src/adapters/catalog_decoder.rs new file mode 100644 index 0000000..3a6f4c8 --- /dev/null +++ b/src/adapters/catalog_decoder.rs @@ -0,0 +1,173 @@ +//! Canonical catalog header and admission pipeline. + +use super::{ + CatalogDecodeError, ChecksummedCatalog, catalog_entry_sequence, catalog_header_decoder, + catalog_integrity, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +const MAGIC: [u8; 16] = *b"KEEP:CATALOG:V1\0"; +const VERSION: u16 = 1; +const FLAGS: u16 = 0; +const MAXIMUM_ENTRY_COUNT: u64 = 1_048_576; +const ALGORITHM: u8 = 1; + +pub(super) fn decode(encoded: &[u8]) -> Result, CatalogDecodeError> { + let fields = catalog_header_decoder::decode(encoded)?; + let (generation, predecessor, catalog_length) = validate_header(&fields)?; + validate_observed_length(encoded, catalog_length)?; + catalog_entry_sequence::validate(encoded, fields.entry_count)?; + let digest = catalog_integrity::validate(encoded)?; + Ok(ChecksummedCatalog::from_verified_parts( + encoded, + generation, + predecessor, + fields.entry_count, + CatalogDigest::from_validated(digest), + )) +} + +fn validate_header( + fields: &catalog_header_decoder::DecodedCatalogHeader, +) -> Result<(CatalogGeneration, Option, CatalogLength), CatalogDecodeError> { + validate_fixed_fields(fields)?; + let generation = CatalogGeneration::new(fields.generation) + .map_err(|source| CatalogDecodeError::Generation { source })?; + let predecessor = validate_predecessor(generation, fields.previous_digest)?; + if fields.entry_count > MAXIMUM_ENTRY_COUNT { + return Err(CatalogDecodeError::EntryCountOutOfBounds { + maximum: MAXIMUM_ENTRY_COUNT, + observed: fields.entry_count, + }); + } + let catalog_length = CatalogLength::new(fields.catalog_length) + .map_err(|source| CatalogDecodeError::CatalogLength { source })?; + validate_count_length(fields.entry_count, catalog_length)?; + Ok((generation, predecessor, catalog_length)) +} + +fn validate_fixed_fields( + fields: &catalog_header_decoder::DecodedCatalogHeader, +) -> Result<(), CatalogDecodeError> { + require_eq(fields.magic, MAGIC, |observed| { + CatalogDecodeError::InvalidMagic { observed } + })?; + require_eq(fields.version, VERSION, |observed| { + CatalogDecodeError::UnsupportedVersion { + expected: VERSION, + observed, + } + })?; + require_eq(fields.flags, FLAGS, |observed| CatalogDecodeError::Flags { + expected: FLAGS, + observed, + })?; + require_eq( + fields.header_length, + catalog_header_decoder::HEADER_LENGTH, + |observed| CatalogDecodeError::HeaderLength { + expected: catalog_header_decoder::HEADER_LENGTH, + observed, + }, + )?; + require_eq( + fields.entry_length, + catalog_header_decoder::ENTRY_LENGTH, + |observed| CatalogDecodeError::EntryLength { + expected: catalog_header_decoder::ENTRY_LENGTH, + observed, + }, + )?; + require_eq(fields.checksum_algorithm, ALGORITHM, |observed| { + CatalogDecodeError::ChecksumAlgorithm { + expected: ALGORITHM, + observed, + } + })?; + require_eq(fields.digest_algorithm, ALGORITHM, |observed| { + CatalogDecodeError::DigestAlgorithm { + expected: ALGORITHM, + observed, + } + })?; + let expected = [0_u8; 46]; + require_eq(fields.reserved, expected, |observed| { + CatalogDecodeError::Reserved { expected, observed } + }) +} + +fn validate_predecessor( + generation: CatalogGeneration, + observed: [u8; 32], +) -> Result, CatalogDecodeError> { + let zero = [0_u8; 32]; + if generation.get() == 1 { + return if observed == zero { + Ok(None) + } else { + Err(CatalogDecodeError::UnexpectedPredecessor { + generation: generation.get(), + observed, + }) + }; + } + if observed == zero { + return Err(CatalogDecodeError::MissingPredecessor { + generation: generation.get(), + }); + } + Ok(Some(CatalogDigest::from_validated(observed))) +} + +fn validate_count_length( + entry_count: u64, + observed: CatalogLength, +) -> Result<(), CatalogDecodeError> { + let expected = entry_count + .checked_mul(u64::from(catalog_header_decoder::ENTRY_LENGTH)) + .and_then(|bytes| bytes.checked_add(u64::from(catalog_header_decoder::HEADER_LENGTH))) + .and_then(|bytes| { + bytes.checked_add(u64::try_from(catalog_header_decoder::TRAILER_LENGTH).ok()?) + }) + .ok_or(CatalogDecodeError::LengthArithmetic { entry_count })?; + if expected == observed.get() { + Ok(()) + } else { + Err(CatalogDecodeError::EntryCountLengthMismatch { + entry_count, + expected, + observed: observed.get(), + }) + } +} + +fn validate_observed_length( + encoded: &[u8], + declared: CatalogLength, +) -> Result<(), CatalogDecodeError> { + let expected = + usize::try_from(declared.get()).map_err(|_source| CatalogDecodeError::ObservedLength { + declared: declared.get(), + observed: encoded.len(), + })?; + if encoded.len() == expected { + Ok(()) + } else { + Err(CatalogDecodeError::ObservedLength { + declared: declared.get(), + observed: encoded.len(), + }) + } +} + +fn require_eq( + observed: T, + expected: T, + error: impl FnOnce(T) -> CatalogDecodeError, +) -> Result<(), CatalogDecodeError> { + if observed == expected { + Ok(()) + } else { + Err(error(observed)) + } +} diff --git a/src/adapters/catalog_entry_decode_error.rs b/src/adapters/catalog_entry_decode_error.rs new file mode 100644 index 0000000..19bf8dc --- /dev/null +++ b/src/adapters/catalog_entry_decode_error.rs @@ -0,0 +1,111 @@ +//! Catalog-entry decoding failures. + +use crate::LayoutIdBinaryParseError; + +/// Failure to admit one fixed-width catalog entry. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogEntryDecodeError { + /// The supplied entry was not exactly the fixed version-1 width. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The record-kind coordinate is unsupported. + UnknownRecordKind { + /// Kind supplied by the entry. + observed: u8, + }, + /// Version-1 entry flags were nonzero. + Flags { + /// Canonical flag field. + expected: u8, + /// Observed flag field. + observed: u8, + }, + /// The identity width disagreed with the record kind. + IdentityLength { + /// Admitted record-kind code. + record_kind: u8, + /// Canonical meaningful identity width. + expected: u16, + /// Observed identity width. + observed: u16, + }, + /// A chunk identity declared the forbidden zero length. + ZeroChunkLength { + /// Observed chunk length. + observed: u32, + }, + /// Unused bytes in a chunk identity slot were nonzero. + NonzeroChunkIdentityTail { + /// Required all-zero tail. + expected: [u8; 24], + /// Observed tail. + observed: [u8; 24], + }, + /// The chunk identity and entry payload lengths disagreed. + ChunkPayloadLengthMismatch { + /// Length committed by the identity. + identity_length: u32, + /// Length declared by the entry. + payload_length: u64, + }, + /// The layout identity was structurally invalid. + LayoutIdentity { + /// Exact nested identity failure. + source: LayoutIdBinaryParseError, + }, + /// The layout identity and entry payload lengths disagreed. + LayoutPayloadLengthMismatch { + /// Length committed by the identity. + identity_length: u64, + /// Length declared by the entry. + payload_length: u64, + }, + /// The payload length exceeded kind-specific protocol bounds. + PayloadLengthOutOfBounds { + /// Smallest lawful payload. + minimum: u64, + /// Largest lawful payload. + maximum: u64, + /// Observed payload. + observed: u64, + }, + /// The top-level record offset preceded the segment header. + RecordOffset { + /// First lawful top-level record offset. + minimum: u64, + /// Observed offset. + observed: u64, + }, + /// The complete-record length did not equal payload plus framing. + RecordLengthMismatch { + /// Declared payload length. + payload_length: u64, + /// Canonical complete-record length. + expected: u64, + /// Observed complete-record length. + observed: u64, + }, + /// Checked complete-record length arithmetic overflowed. + RecordLengthArithmetic { + /// Declared payload length. + payload_length: u64, + }, + /// Checked record-span arithmetic overflowed. + RecordSpanArithmetic { + /// Declared record offset. + record_offset: u64, + /// Declared complete-record length. + record_length: u64, + }, + /// Version-1 reserved bytes were nonzero. + Reserved { + /// Required all-zero bytes. + expected: [u8; 8], + /// Observed reserved bytes. + observed: [u8; 8], + }, +} diff --git a/src/adapters/catalog_entry_decode_error_display.rs b/src/adapters/catalog_entry_decode_error_display.rs new file mode 100644 index 0000000..5d41b00 --- /dev/null +++ b/src/adapters/catalog_entry_decode_error_display.rs @@ -0,0 +1,101 @@ +//! Human-readable catalog-entry decoding diagnostics. + +use std::error::Error; +use std::fmt; + +use super::CatalogEntryDecodeError; + +impl fmt::Display for CatalogEntryDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "wrong catalog-entry length: expected {expected}, observed {observed}" + ), + Self::UnknownRecordKind { observed } => { + write!(formatter, "unknown catalog record kind {observed}") + } + Self::Flags { expected, observed } => write!( + formatter, + "noncanonical catalog-entry flags: expected {expected}, observed {observed}" + ), + Self::IdentityLength { + record_kind, + expected, + observed, + } => write!( + formatter, + "wrong kind-{record_kind} identity length: expected {expected}, observed {observed}" + ), + Self::ZeroChunkLength { observed } => { + write!( + formatter, + "catalog chunk length must be positive, observed {observed}" + ) + } + Self::NonzeroChunkIdentityTail { .. } => { + formatter.write_str("nonzero catalog chunk identity tail") + } + Self::ChunkPayloadLengthMismatch { + identity_length, + payload_length, + } => write!( + formatter, + "catalog chunk identity length {identity_length} disagrees with payload {payload_length}" + ), + Self::LayoutIdentity { source } => { + write!(formatter, "invalid catalog layout identity: {source}") + } + Self::LayoutPayloadLengthMismatch { + identity_length, + payload_length, + } => write!( + formatter, + "catalog layout identity length {identity_length} disagrees with payload {payload_length}" + ), + Self::PayloadLengthOutOfBounds { + minimum, + maximum, + observed, + } => write!( + formatter, + "catalog payload length {observed} is outside {minimum}..={maximum}" + ), + Self::RecordOffset { minimum, observed } => { + write!( + formatter, + "catalog record offset {observed} precedes {minimum}" + ) + } + Self::RecordLengthMismatch { + payload_length, + expected, + observed, + } => write!( + formatter, + "catalog payload {payload_length} requires record length {expected}, observed {observed}" + ), + Self::RecordLengthArithmetic { payload_length } => write!( + formatter, + "catalog record-length arithmetic overflowed for payload {payload_length}" + ), + Self::RecordSpanArithmetic { + record_offset, + record_length, + } => write!( + formatter, + "catalog record span overflows at {record_offset} plus {record_length}" + ), + Self::Reserved { .. } => formatter.write_str("nonzero catalog-entry reserved bytes"), + } + } +} + +impl Error for CatalogEntryDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LayoutIdentity { source } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/catalog_entry_decoder.rs b/src/adapters/catalog_entry_decoder.rs new file mode 100644 index 0000000..814ea97 --- /dev/null +++ b/src/adapters/catalog_entry_decoder.rs @@ -0,0 +1,173 @@ +//! Canonical fixed-width catalog-entry decoder. + +use super::{ + CatalogEntryDecodeError, SegmentRecordIdentity, + catalog_entry_fields::{self, read_array, read_u8, read_u16, read_u64}, +}; +use crate::{ChunkId, ChunkLength, LayoutId}; + +pub(super) const ENCODED_LENGTH: usize = catalog_entry_fields::ENCODED_LENGTH; +const FLAGS: u8 = 0; +const CHUNK_KIND: u8 = 1; +const LAYOUT_KIND: u8 = 2; +const CHUNK_IDENTITY_LENGTH: u16 = 36; +const LAYOUT_IDENTITY_LENGTH: u16 = 60; +const SEGMENT_HEADER_LENGTH: u64 = 64; +const RECORD_FRAMING_LENGTH: u64 = 144; +const MAXIMUM_RECORD_PAYLOAD_LENGTH: u64 = 67_108_864; + +pub(super) fn decode(encoded: &[u8]) -> Result { + if encoded.len() != ENCODED_LENGTH { + return Err(CatalogEntryDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }); + } + let kind = read_u8(encoded, 0)?; + let flags = read_u8(encoded, 1)?; + if flags != FLAGS { + return Err(CatalogEntryDecodeError::Flags { + expected: FLAGS, + observed: flags, + }); + } + let identity_length = read_u16(encoded, 2)?; + let identity_slot = read_array(encoded, 4)?; + let record_offset = read_u64(encoded, 96)?; + let record_length = read_u64(encoded, 104)?; + let payload_length = read_u64(encoded, 112)?; + let reserved = read_array(encoded, 152)?; + let identity = decode_identity(kind, identity_length, identity_slot, payload_length)?; + validate_payload_bounds(kind, payload_length)?; + validate_location(record_offset, record_length, payload_length)?; + let expected = [0_u8; 8]; + if reserved != expected { + return Err(CatalogEntryDecodeError::Reserved { + expected, + observed: reserved, + }); + } + Ok(identity) +} + +fn decode_identity( + kind: u8, + identity_length: u16, + identity: [u8; 60], + payload_length: u64, +) -> Result { + match kind { + CHUNK_KIND => decode_chunk(identity_length, identity, payload_length), + LAYOUT_KIND => decode_layout(identity_length, identity, payload_length), + observed => Err(CatalogEntryDecodeError::UnknownRecordKind { observed }), + } +} + +fn decode_chunk( + identity_length: u16, + identity: [u8; 60], + payload_length: u64, +) -> Result { + require_identity_length(CHUNK_KIND, CHUNK_IDENTITY_LENGTH, identity_length)?; + let length_bytes = read_array(&identity, 0)?; + let digest = read_array(&identity, 4)?; + let observed_tail = read_array(&identity, 36)?; + let expected_tail = [0_u8; 24]; + if observed_tail != expected_tail { + return Err(CatalogEntryDecodeError::NonzeroChunkIdentityTail { + expected: expected_tail, + observed: observed_tail, + }); + } + let length_value = u32::from_be_bytes(length_bytes); + let length = + ChunkLength::from_wire(length_value).ok_or(CatalogEntryDecodeError::ZeroChunkLength { + observed: length_value, + })?; + if u64::from(length_value) != payload_length { + return Err(CatalogEntryDecodeError::ChunkPayloadLengthMismatch { + identity_length: length_value, + payload_length, + }); + } + Ok(SegmentRecordIdentity::Chunk(ChunkId::from_validated_parts( + length, digest, + ))) +} + +fn decode_layout( + identity_length: u16, + identity: [u8; 60], + payload_length: u64, +) -> Result { + require_identity_length(LAYOUT_KIND, LAYOUT_IDENTITY_LENGTH, identity_length)?; + let layout = LayoutId::parse_binary(&identity) + .map_err(|source| CatalogEntryDecodeError::LayoutIdentity { source })?; + let identity_length = layout.plan_length().get(); + if identity_length != payload_length { + return Err(CatalogEntryDecodeError::LayoutPayloadLengthMismatch { + identity_length, + payload_length, + }); + } + Ok(SegmentRecordIdentity::Layout(layout)) +} + +const fn require_identity_length( + record_kind: u8, + expected: u16, + observed: u16, +) -> Result<(), CatalogEntryDecodeError> { + if observed == expected { + Ok(()) + } else { + Err(CatalogEntryDecodeError::IdentityLength { + record_kind, + expected, + observed, + }) + } +} + +const fn validate_payload_bounds(kind: u8, observed: u64) -> Result<(), CatalogEntryDecodeError> { + let minimum = 1; + let maximum = MAXIMUM_RECORD_PAYLOAD_LENGTH; + if matches!(kind, CHUNK_KIND | LAYOUT_KIND) && (observed < minimum || observed > maximum) { + return Err(CatalogEntryDecodeError::PayloadLengthOutOfBounds { + minimum, + maximum, + observed, + }); + } + Ok(()) +} + +fn validate_location( + record_offset: u64, + record_length: u64, + payload_length: u64, +) -> Result<(), CatalogEntryDecodeError> { + if record_offset < SEGMENT_HEADER_LENGTH { + return Err(CatalogEntryDecodeError::RecordOffset { + minimum: SEGMENT_HEADER_LENGTH, + observed: record_offset, + }); + } + let expected = payload_length + .checked_add(RECORD_FRAMING_LENGTH) + .ok_or(CatalogEntryDecodeError::RecordLengthArithmetic { payload_length })?; + if record_length != expected { + return Err(CatalogEntryDecodeError::RecordLengthMismatch { + payload_length, + expected, + observed: record_length, + }); + } + record_offset.checked_add(record_length).ok_or( + CatalogEntryDecodeError::RecordSpanArithmetic { + record_offset, + record_length, + }, + )?; + Ok(()) +} diff --git a/src/adapters/catalog_entry_fields.rs b/src/adapters/catalog_entry_fields.rs new file mode 100644 index 0000000..9f68856 --- /dev/null +++ b/src/adapters/catalog_entry_fields.rs @@ -0,0 +1,40 @@ +//! Bounded fixed-field extraction for one complete catalog entry. + +use super::CatalogEntryDecodeError; + +pub(super) const ENCODED_LENGTH: usize = 160; + +pub(super) fn read_u8(encoded: &[u8], offset: usize) -> Result { + encoded + .get(offset) + .copied() + .ok_or_else(|| wrong_length(encoded)) +} + +pub(super) fn read_u16(encoded: &[u8], offset: usize) -> Result { + Ok(u16::from_be_bytes(read_array(encoded, offset)?)) +} + +pub(super) fn read_u64(encoded: &[u8], offset: usize) -> Result { + Ok(u64::from_be_bytes(read_array(encoded, offset)?)) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; LENGTH], CatalogEntryDecodeError> { + let end = offset + .checked_add(LENGTH) + .ok_or_else(|| wrong_length(encoded))?; + encoded + .get(offset..end) + .and_then(|field| field.try_into().ok()) + .ok_or_else(|| wrong_length(encoded)) +} + +const fn wrong_length(encoded: &[u8]) -> CatalogEntryDecodeError { + CatalogEntryDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + } +} diff --git a/src/adapters/catalog_entry_sequence.rs b/src/adapters/catalog_entry_sequence.rs new file mode 100644 index 0000000..198f457 --- /dev/null +++ b/src/adapters/catalog_entry_sequence.rs @@ -0,0 +1,57 @@ +//! Streaming canonical-order validation for catalog entries. + +use std::cmp::Ordering; + +use super::{ + CatalogDecodeError, SegmentRecordIdentity, catalog_entry_decoder, catalog_header_decoder, +}; + +pub(super) fn validate(encoded: &[u8], entry_count: u64) -> Result<(), CatalogDecodeError> { + let entries_end = encoded + .len() + .checked_sub(catalog_header_decoder::TRAILER_LENGTH) + .ok_or(CatalogDecodeError::MinimumLength { + minimum: catalog_header_decoder::MINIMUM_LENGTH, + observed: encoded.len(), + })?; + let entries = encoded + .get(catalog_header_decoder::HEADER_LENGTH_BYTES..entries_end) + .ok_or(CatalogDecodeError::MinimumLength { + minimum: catalog_header_decoder::MINIMUM_LENGTH, + observed: encoded.len(), + })?; + let mut previous: Option<(u64, SegmentRecordIdentity)> = None; + for (host_index, entry) in entries + .chunks_exact(catalog_entry_decoder::ENCODED_LENGTH) + .enumerate() + { + let index = u64::try_from(host_index) + .map_err(|_source| CatalogDecodeError::LengthArithmetic { entry_count })?; + let identity = catalog_entry_decoder::decode(entry) + .map_err(|source| CatalogDecodeError::Entry { index, source })?; + validate_order(previous, index, identity)?; + previous = Some((index, identity)); + } + Ok(()) +} + +fn validate_order( + previous: Option<(u64, SegmentRecordIdentity)>, + observed_index: u64, + observed: SegmentRecordIdentity, +) -> Result<(), CatalogDecodeError> { + let Some((previous_index, previous_identity)) = previous else { + return Ok(()); + }; + match previous_identity.cmp(&observed) { + Ordering::Less => Ok(()), + Ordering::Equal => Err(CatalogDecodeError::DuplicateIdentity { + first_index: previous_index, + duplicate_index: observed_index, + }), + Ordering::Greater => Err(CatalogDecodeError::IdentityOrder { + previous_index, + observed_index, + }), + } +} diff --git a/src/adapters/catalog_header_decoder.rs b/src/adapters/catalog_header_decoder.rs new file mode 100644 index 0000000..55c0e8a --- /dev/null +++ b/src/adapters/catalog_header_decoder.rs @@ -0,0 +1,76 @@ +//! Fixed-width version-1 catalog-header field decoder. + +use super::CatalogDecodeError; + +pub(super) const HEADER_LENGTH: u16 = 128; +pub(super) const HEADER_LENGTH_BYTES: usize = 128; +pub(super) const ENTRY_LENGTH: u16 = 160; +pub(super) const TRAILER_LENGTH: usize = 64; +pub(super) const MINIMUM_LENGTH: usize = HEADER_LENGTH_BYTES + TRAILER_LENGTH; + +pub(super) struct DecodedCatalogHeader { + pub(super) magic: [u8; 16], + pub(super) version: u16, + pub(super) flags: u16, + pub(super) header_length: u16, + pub(super) entry_length: u16, + pub(super) generation: u64, + pub(super) previous_digest: [u8; 32], + pub(super) entry_count: u64, + pub(super) catalog_length: u64, + pub(super) checksum_algorithm: u8, + pub(super) digest_algorithm: u8, + pub(super) reserved: [u8; 46], +} + +pub(super) fn decode(encoded: &[u8]) -> Result { + if encoded.len() < MINIMUM_LENGTH { + return Err(CatalogDecodeError::MinimumLength { + minimum: MINIMUM_LENGTH, + observed: encoded.len(), + }); + } + Ok(DecodedCatalogHeader { + magic: read_array(encoded, 0)?, + version: u16::from_be_bytes(read_array(encoded, 16)?), + flags: u16::from_be_bytes(read_array(encoded, 18)?), + header_length: u16::from_be_bytes(read_array(encoded, 20)?), + entry_length: u16::from_be_bytes(read_array(encoded, 22)?), + generation: u64::from_be_bytes(read_array(encoded, 24)?), + previous_digest: read_array(encoded, 32)?, + entry_count: u64::from_be_bytes(read_array(encoded, 64)?), + catalog_length: u64::from_be_bytes(read_array(encoded, 72)?), + checksum_algorithm: read_u8(encoded, 80)?, + digest_algorithm: read_u8(encoded, 81)?, + reserved: read_array(encoded, 82)?, + }) +} + +fn read_u8(encoded: &[u8], offset: usize) -> Result { + encoded + .get(offset) + .copied() + .ok_or(CatalogDecodeError::MinimumLength { + minimum: MINIMUM_LENGTH, + observed: encoded.len(), + }) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; LENGTH], CatalogDecodeError> { + let end = offset + .checked_add(LENGTH) + .ok_or(CatalogDecodeError::MinimumLength { + minimum: MINIMUM_LENGTH, + observed: encoded.len(), + })?; + encoded + .get(offset..end) + .and_then(|field| field.try_into().ok()) + .ok_or(CatalogDecodeError::MinimumLength { + minimum: MINIMUM_LENGTH, + observed: encoded.len(), + }) +} diff --git a/src/adapters/catalog_integrity.rs b/src/adapters/catalog_integrity.rs new file mode 100644 index 0000000..3712a31 --- /dev/null +++ b/src/adapters/catalog_integrity.rs @@ -0,0 +1,71 @@ +//! Canonical catalog checksum and physical digest verification. + +use super::{CatalogDecodeError, catalog_header_decoder, framed_blake3}; + +const CHECKSUM_DOMAIN: &[u8] = b"KEEP:CATALOG:SUM\0"; +const DIGEST_DOMAIN: &[u8] = b"KEEP:CATALOG:DIGEST\0"; + +pub(super) fn validate(encoded: &[u8]) -> Result<[u8; 32], CatalogDecodeError> { + let checksum_offset = encoded + .len() + .checked_sub(catalog_header_decoder::TRAILER_LENGTH) + .ok_or_else(|| minimum_length(encoded))?; + let digest_offset = encoded + .len() + .checked_sub(32) + .ok_or_else(|| minimum_length(encoded))?; + let covered = encoded + .get(..checksum_offset) + .ok_or_else(|| minimum_length(encoded))?; + let observed_checksum = read_array(encoded, checksum_offset)?; + let expected_checksum = + framed_blake3::hash(CHECKSUM_DOMAIN, &[covered], admitted_length(covered)?); + if observed_checksum != expected_checksum { + return Err(CatalogDecodeError::ChecksumMismatch { + expected: expected_checksum, + observed: observed_checksum, + }); + } + let observed_digest = read_array(encoded, digest_offset)?; + let digest_input = encoded + .get(..digest_offset) + .ok_or_else(|| minimum_length(encoded))?; + let expected_digest = framed_blake3::hash( + DIGEST_DOMAIN, + &[digest_input], + admitted_length(digest_input)?, + ); + if observed_digest != expected_digest { + return Err(CatalogDecodeError::DigestMismatch { + expected: expected_digest, + observed: observed_digest, + }); + } + Ok(observed_digest) +} + +fn admitted_length(bytes: &[u8]) -> Result { + u64::try_from(bytes.len()).map_err(|_source| CatalogDecodeError::HashLength { + observed: bytes.len(), + }) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; LENGTH], CatalogDecodeError> { + let end = offset + .checked_add(LENGTH) + .ok_or_else(|| minimum_length(encoded))?; + encoded + .get(offset..end) + .and_then(|field| field.try_into().ok()) + .ok_or_else(|| minimum_length(encoded)) +} + +const fn minimum_length(encoded: &[u8]) -> CatalogDecodeError { + CatalogDecodeError::MinimumLength { + minimum: catalog_header_decoder::MINIMUM_LENGTH, + observed: encoded.len(), + } +} diff --git a/src/adapters/checksummed_catalog.rs b/src/adapters/checksummed_catalog.rs new file mode 100644 index 0000000..2b7494a --- /dev/null +++ b/src/adapters/checksummed_catalog.rs @@ -0,0 +1,77 @@ +//! Canonically framed, checksum- and digest-verified borrowed catalog. + +use super::{CatalogDecodeError, catalog_decoder}; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Borrowed catalog bytes with canonical framing, ordering, and integrity proof. +/// +/// This state validates catalog-local fields and entry coordinates. It does not +/// prove that any named segment exists or that a location selects a top-level +/// admitted record. Callers must not treat it as a reader snapshot. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ChecksummedCatalog<'a> { + encoded: &'a [u8], + generation: CatalogGeneration, + previous_catalog_digest: Option, + entry_count: u64, + digest: CatalogDigest, +} + +impl<'a> ChecksummedCatalog<'a> { + /// Decodes one exact version-1 catalog and verifies local integrity. + /// + /// Admission streams fixed-width entries without heap allocation or I/O. + /// + /// # Errors + /// + /// Returns [`CatalogDecodeError`] for malformed, unsupported, + /// noncanonical, unordered, duplicate, corrupt, or over-limit bytes. + pub fn decode(encoded: &'a [u8]) -> Result { + catalog_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(self) -> &'a [u8] { + self.encoded + } + + /// Returns the positive catalog generation. + pub const fn generation(self) -> CatalogGeneration { + self.generation + } + + /// Returns the predecessor witness, absent only for generation 1. + #[must_use] + pub const fn previous_catalog_digest(self) -> Option { + self.previous_catalog_digest + } + + /// Returns the exact bounded entry count. + #[must_use] + pub const fn entry_count(self) -> u64 { + self.entry_count + } + + /// Returns the verified physical catalog digest. + pub const fn digest(self) -> CatalogDigest { + self.digest + } + + pub(super) const fn from_verified_parts( + encoded: &'a [u8], + generation: CatalogGeneration, + previous_catalog_digest: Option, + entry_count: u64, + digest: CatalogDigest, + ) -> Self { + Self { + encoded, + generation, + previous_catalog_digest, + entry_count, + digest, + } + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 1b0d38e..3502ba7 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -11,6 +11,17 @@ mod blob_id_binary; mod blob_id_binary_error; mod blob_id_text; mod blob_id_text_error; +mod catalog_decode_error; +mod catalog_decode_error_display; +mod catalog_decoder; +mod catalog_entry_decode_error; +mod catalog_entry_decode_error_display; +mod catalog_entry_decoder; +mod catalog_entry_fields; +mod catalog_entry_sequence; +mod catalog_header_decoder; +mod catalog_integrity; +mod checksummed_catalog; mod checksummed_publication_head; mod checksummed_segment_record; mod filesystem_segment_stage; @@ -94,6 +105,9 @@ pub use admitted_segment::AdmittedSegment; pub use admitted_segment_record::AdmittedSegmentRecord; pub use blob_id_binary_error::BlobIdBinaryParseError; pub use blob_id_text_error::BlobIdTextParseError; +pub use catalog_decode_error::CatalogDecodeError; +pub use catalog_entry_decode_error::CatalogEntryDecodeError; +pub use checksummed_catalog::ChecksummedCatalog; pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; pub use filesystem_segment_stage::FilesystemSegmentStage; diff --git a/src/adapters/segment_record_checksum.rs b/src/adapters/segment_record_checksum.rs index a5b4655..249cfca 100644 --- a/src/adapters/segment_record_checksum.rs +++ b/src/adapters/segment_record_checksum.rs @@ -1,12 +1,8 @@ //! Typed segment-record checksum and canonical calculation. -use blake3::Hasher; - -use super::SegmentRecordHeader; +use super::{SegmentRecordHeader, framed_blake3}; const DOMAIN: &[u8] = b"KEEP:SEG:RECORD:SUM\0"; -const FRAMING_VERSION: u16 = 1; -const ALGORITHM: u8 = 1; /// BLAKE3-256 checksum binding one complete segment-record header and payload. #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] @@ -24,14 +20,11 @@ impl SegmentRecordChecksum { payload: &[u8], covered_length: u64, ) -> Self { - let mut hasher = Hasher::new(); - hasher.update(DOMAIN); - hasher.update(&FRAMING_VERSION.to_be_bytes()); - hasher.update(&[ALGORITHM]); - hasher.update(&header.encode()); - hasher.update(payload); - hasher.update(&covered_length.to_be_bytes()); - Self(*hasher.finalize().as_bytes()) + Self(framed_blake3::hash( + DOMAIN, + &[&header.encode(), payload], + covered_length, + )) } pub(super) const fn from_validated(bytes: [u8; 32]) -> Self { diff --git a/src/lib.rs b/src/lib.rs index 82b708f..6e22368 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,15 +21,16 @@ mod reference; pub use adapters::{ AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalLayoutRecord, ChecksummedPublicationHead, ChecksummedSegmentRecord, - FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, - SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + CanonicalLayoutRecord, CatalogDecodeError, CatalogEntryDecodeError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemSegmentStage, + LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, + LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, + SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, + SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, + SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, + SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, + SegmentWritePhase, StagedSegment, StorageProfileIdParseError, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog.rs b/tests/catalog.rs new file mode 100644 index 0000000..f19be0b --- /dev/null +++ b/tests/catalog.rs @@ -0,0 +1,93 @@ +//! Public catalog framing, ordering, checksum, and digest laws. + +#[path = "catalog/entry_laws.rs"] +mod entry_laws; +#[path = "catalog/format_oracle.rs"] +mod format_oracle; +#[path = "catalog/header_laws.rs"] +mod header_laws; +#[path = "catalog/integrity_laws.rs"] +mod integrity_laws; +#[path = "catalog/mutation_support.rs"] +mod mutation_support; +#[path = "catalog/ordering_laws.rs"] +mod ordering_laws; +mod support; + +use std::error::Error; + +use keep::ChecksummedCatalog; +use support::decode_hex; + +pub(crate) const GENERATION_ONE_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +pub(crate) const GENERATION_TWO_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +pub(crate) const BUNDLE_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const GENERATION_ONE_DIGEST_HEX: &str = + "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320"; +const GENERATION_TWO_DIGEST_HEX: &str = + "ea7d0055fd21f00ed94809ef4e671d72fa2e6a4a5d9ecefb23f3a320a2dad993"; +const BUNDLE_DIGEST_HEX: &str = "0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811"; + +pub(crate) const VERSION_OFFSET: usize = 16; +pub(crate) const FLAGS_OFFSET: usize = 18; +pub(crate) const HEADER_LENGTH_OFFSET: usize = 20; +pub(crate) const ENTRY_LENGTH_OFFSET: usize = 22; +pub(crate) const GENERATION_OFFSET: usize = 24; +pub(crate) const PREVIOUS_DIGEST_OFFSET: usize = 32; +pub(crate) const ENTRY_COUNT_OFFSET: usize = 64; +pub(crate) const CATALOG_LENGTH_OFFSET: usize = 72; +pub(crate) const CHECKSUM_ALGORITHM_OFFSET: usize = 80; +pub(crate) const DIGEST_ALGORITHM_OFFSET: usize = 81; +pub(crate) const HEADER_RESERVED_OFFSET: usize = 82; +pub(crate) const FIRST_ENTRY_OFFSET: usize = 128; +pub(crate) const ENTRY_FLAGS_OFFSET: usize = FIRST_ENTRY_OFFSET + 1; +pub(crate) const ENTRY_IDENTITY_LENGTH_OFFSET: usize = FIRST_ENTRY_OFFSET + 2; +pub(crate) const ENTRY_RECORD_OFFSET: usize = FIRST_ENTRY_OFFSET + 96; +pub(crate) const ENTRY_RECORD_LENGTH_OFFSET: usize = FIRST_ENTRY_OFFSET + 104; +pub(crate) const ENTRY_PAYLOAD_LENGTH_OFFSET: usize = FIRST_ENTRY_OFFSET + 112; +pub(crate) const ENTRY_RESERVED_OFFSET: usize = FIRST_ENTRY_OFFSET + 152; +pub(crate) const ENTRY_LENGTH: usize = 160; + +#[test] +fn frozen_catalogs_are_checksum_and_digest_verified_exactly() -> Result<(), Box> { + let generation_one = catalog_bytes(GENERATION_ONE_HEX)?; + let first = ChecksummedCatalog::decode(&generation_one)?; + assert_eq!(first.generation().get(), 1); + assert_eq!(first.previous_catalog_digest(), None); + assert_eq!(first.entry_count(), 1); + assert_eq!( + first.digest().as_bytes().as_slice(), + decode_hex(GENERATION_ONE_DIGEST_HEX)? + ); + assert_eq!(first.encoded(), generation_one); + + let generation_two = catalog_bytes(GENERATION_TWO_HEX)?; + let second = ChecksummedCatalog::decode(&generation_two)?; + assert_eq!(second.generation().get(), 2); + assert_eq!(second.previous_catalog_digest(), Some(first.digest())); + assert_eq!( + second.digest().as_bytes().as_slice(), + decode_hex(GENERATION_TWO_DIGEST_HEX)? + ); + + let bundle = catalog_bytes(BUNDLE_HEX)?; + let bundle_catalog = ChecksummedCatalog::decode(&bundle)?; + assert_eq!(bundle_catalog.generation().get(), 1); + assert_eq!(bundle_catalog.entry_count(), 2); + assert_eq!( + bundle_catalog.digest().as_bytes().as_slice(), + decode_hex(BUNDLE_DIGEST_HEX)? + ); + Ok(()) +} + +pub(crate) fn catalog_bytes(hex: &str) -> Result, Box> { + decode_hex( + hex.strip_suffix('\n') + .ok_or("catalog fixture must end in one LF")?, + ) + .map_err(Into::into) +} diff --git a/tests/catalog/entry_laws.rs b/tests/catalog/entry_laws.rs new file mode 100644 index 0000000..001d4f7 --- /dev/null +++ b/tests/catalog/entry_laws.rs @@ -0,0 +1,85 @@ +//! Catalog-entry field and location-coordinate laws. + +use std::error::Error; + +use keep::{CatalogDecodeError, CatalogEntryDecodeError}; + +use super::{ + ENTRY_FLAGS_OFFSET, ENTRY_IDENTITY_LENGTH_OFFSET, ENTRY_PAYLOAD_LENGTH_OFFSET, + ENTRY_RECORD_LENGTH_OFFSET, ENTRY_RECORD_OFFSET, ENTRY_RESERVED_OFFSET, FIRST_ENTRY_OFFSET, + mutation_support, +}; + +#[test] +fn catalog_refuses_noncanonical_entry_fields_and_lengths() -> Result<(), Box> { + mutation_support::assert_byte_refusal(FIRST_ENTRY_OFFSET, 3, |error| { + error + == CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::UnknownRecordKind { observed: 3 }, + } + })?; + mutation_support::assert_byte_refusal(ENTRY_FLAGS_OFFSET, 1, |error| { + error + == CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::Flags { + expected: 0, + observed: 1, + }, + } + })?; + mutation_support::assert_u16_refusal(ENTRY_IDENTITY_LENGTH_OFFSET, 35, |error| { + error + == CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::IdentityLength { + record_kind: 1, + expected: 36, + observed: 35, + }, + } + })?; + mutation_support::assert_u64_refusal(ENTRY_RECORD_OFFSET, 63, |error| { + error + == CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::RecordOffset { + minimum: 64, + observed: 63, + }, + } + })?; + mutation_support::assert_u64_refusal(ENTRY_RECORD_LENGTH_OFFSET, 144, |error| { + error + == CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::RecordLengthMismatch { + payload_length: 1, + expected: 145, + observed: 144, + }, + } + })?; + mutation_support::assert_u64_refusal(ENTRY_PAYLOAD_LENGTH_OFFSET, 2, |error| { + matches!( + error, + CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::ChunkPayloadLengthMismatch { + identity_length: 1, + payload_length: 2, + }, + } + ) + })?; + mutation_support::assert_byte_refusal(ENTRY_RESERVED_OFFSET, 1, |error| { + matches!( + error, + CatalogDecodeError::Entry { + index: 0, + source: CatalogEntryDecodeError::Reserved { .. }, + } + ) + }) +} diff --git a/tests/catalog/format_oracle.rs b/tests/catalog/format_oracle.rs new file mode 100644 index 0000000..944f7e7 --- /dev/null +++ b/tests/catalog/format_oracle.rs @@ -0,0 +1,58 @@ +//! Independent catalog checksum and digest test oracle. +#![allow( + clippy::redundant_pub_crate, + reason = "the sibling mutation module consumes this private test oracle" +)] + +use std::error::Error; + +use blake3::Hasher; + +const VERSION: u16 = 1; +const ALGORITHM: u8 = 1; +const TRAILER_LENGTH: usize = 64; +const DIGEST_LENGTH: usize = 32; +const CHECKSUM_DOMAIN: &[u8] = b"KEEP:CATALOG:SUM\0"; +const DIGEST_DOMAIN: &[u8] = b"KEEP:CATALOG:DIGEST\0"; + +pub(crate) fn seal(encoded: &mut [u8]) -> Result<(), Box> { + let checksum_offset = encoded + .len() + .checked_sub(TRAILER_LENGTH) + .ok_or("test catalog lacks its trailer")?; + let digest_offset = encoded + .len() + .checked_sub(DIGEST_LENGTH) + .ok_or("test catalog lacks its digest")?; + let checksum = framed_hash( + CHECKSUM_DOMAIN, + encoded + .get(..checksum_offset) + .ok_or("test catalog lacks checksum input")?, + )?; + encoded + .get_mut(checksum_offset..digest_offset) + .ok_or("test catalog lacks checksum field")? + .copy_from_slice(&checksum); + let digest = framed_hash( + DIGEST_DOMAIN, + encoded + .get(..digest_offset) + .ok_or("test catalog lacks digest input")?, + )?; + encoded + .get_mut(digest_offset..) + .ok_or("test catalog lacks digest field")? + .copy_from_slice(&digest); + Ok(()) +} + +fn framed_hash(domain: &[u8], input: &[u8]) -> Result<[u8; 32], Box> { + let mut hasher = Hasher::new(); + hasher.update(domain); + hasher.update(&VERSION.to_be_bytes()); + hasher.update(&[ALGORITHM]); + hasher.update(input); + hasher.update(&u64::try_from(input.len())?.to_be_bytes()); + Ok(*hasher.finalize().as_bytes()) +} diff --git a/tests/catalog/header_laws.rs b/tests/catalog/header_laws.rs new file mode 100644 index 0000000..bf92a79 --- /dev/null +++ b/tests/catalog/header_laws.rs @@ -0,0 +1,117 @@ +//! Catalog-header canonicalization and predecessor laws. + +use std::error::Error; + +use keep::{CatalogDecodeError, CatalogGenerationError, CatalogLengthError}; + +use super::{ + CATALOG_LENGTH_OFFSET, CHECKSUM_ALGORITHM_OFFSET, DIGEST_ALGORITHM_OFFSET, ENTRY_COUNT_OFFSET, + ENTRY_LENGTH_OFFSET, FLAGS_OFFSET, GENERATION_OFFSET, GENERATION_TWO_HEX, HEADER_LENGTH_OFFSET, + HEADER_RESERVED_OFFSET, PREVIOUS_DIGEST_OFFSET, VERSION_OFFSET, catalog_bytes, + mutation_support, +}; + +#[test] +fn catalog_refuses_noncanonical_header_fields() -> Result<(), Box> { + mutation_support::assert_byte_refusal(0, 0, |error| { + matches!(error, CatalogDecodeError::InvalidMagic { .. }) + })?; + mutation_support::assert_byte_refusal(VERSION_OFFSET + 1, 2, |error| { + error + == CatalogDecodeError::UnsupportedVersion { + expected: 1, + observed: 2, + } + })?; + mutation_support::assert_u16_refusal(FLAGS_OFFSET, 1, |error| { + error + == CatalogDecodeError::Flags { + expected: 0, + observed: 1, + } + })?; + mutation_support::assert_u16_refusal(HEADER_LENGTH_OFFSET, 127, |error| { + error + == CatalogDecodeError::HeaderLength { + expected: 128, + observed: 127, + } + })?; + mutation_support::assert_u16_refusal(ENTRY_LENGTH_OFFSET, 159, |error| { + error + == CatalogDecodeError::EntryLength { + expected: 160, + observed: 159, + } + })?; + mutation_support::assert_byte_refusal(CHECKSUM_ALGORITHM_OFFSET, 2, |error| { + error + == CatalogDecodeError::ChecksumAlgorithm { + expected: 1, + observed: 2, + } + })?; + mutation_support::assert_byte_refusal(DIGEST_ALGORITHM_OFFSET, 2, |error| { + error + == CatalogDecodeError::DigestAlgorithm { + expected: 1, + observed: 2, + } + })?; + mutation_support::assert_byte_refusal(HEADER_RESERVED_OFFSET, 1, |error| { + matches!(error, CatalogDecodeError::Reserved { .. }) + }) +} + +#[test] +fn catalog_refuses_invalid_generation_predecessor_and_length() -> Result<(), Box> { + mutation_support::assert_u64_refusal(GENERATION_OFFSET, 0, |error| { + error + == CatalogDecodeError::Generation { + source: CatalogGenerationError::Zero, + } + })?; + mutation_support::assert_byte_refusal(PREVIOUS_DIGEST_OFFSET, 1, |error| { + matches!( + error, + CatalogDecodeError::UnexpectedPredecessor { generation: 1, .. } + ) + })?; + mutation_support::assert_u64_refusal(ENTRY_COUNT_OFFSET, 1_048_577, |error| { + error + == CatalogDecodeError::EntryCountOutOfBounds { + maximum: 1_048_576, + observed: 1_048_577, + } + })?; + mutation_support::assert_u64_refusal(CATALOG_LENGTH_OFFSET, 191, |error| { + error + == CatalogDecodeError::CatalogLength { + source: CatalogLengthError::OutOfBounds { + minimum: 192, + maximum: 167_772_352, + observed: 191, + }, + } + })?; + mutation_support::assert_u64_refusal(CATALOG_LENGTH_OFFSET, 512, |error| { + error + == CatalogDecodeError::EntryCountLengthMismatch { + entry_count: 1, + expected: 352, + observed: 512, + } + }) +} + +#[test] +fn later_catalog_requires_one_nonzero_predecessor() -> Result<(), Box> { + let mut encoded = catalog_bytes(GENERATION_TWO_HEX)?; + mutation_support::zero_range(&mut encoded, PREVIOUS_DIGEST_OFFSET, 32)?; + mutation_support::assert_catalog_refusal(&mut encoded, |error| { + matches!( + error, + CatalogDecodeError::MissingPredecessor { generation: 2 } + ) + }) +} diff --git a/tests/catalog/integrity_laws.rs b/tests/catalog/integrity_laws.rs new file mode 100644 index 0000000..3e776da --- /dev/null +++ b/tests/catalog/integrity_laws.rs @@ -0,0 +1,51 @@ +//! Catalog width, checksum, and digest refusal laws. + +use std::error::Error; + +use keep::{CatalogDecodeError, ChecksummedCatalog}; + +use super::{GENERATION_ONE_HEX, catalog_bytes}; +use crate::support::require_error; + +#[test] +fn catalog_refuses_wrong_width_checksum_and_digest() -> Result<(), Box> { + let mut truncated = catalog_bytes(GENERATION_ONE_HEX)?; + let _last = truncated.pop().ok_or("catalog fixture is empty")?; + assert_eq!( + require_error( + ChecksummedCatalog::decode(&truncated), + "truncated catalog was admitted" + )?, + CatalogDecodeError::ObservedLength { + declared: 352, + observed: 351, + } + ); + + let mut checksum = catalog_bytes(GENERATION_ONE_HEX)?; + let checksum_offset = checksum.len().checked_sub(64).ok_or("catalog too short")?; + *checksum + .get_mut(checksum_offset) + .ok_or("catalog lacks checksum")? ^= 1; + assert!(matches!( + require_error( + ChecksummedCatalog::decode(&checksum), + "corrupt catalog checksum was admitted" + )?, + CatalogDecodeError::ChecksumMismatch { .. } + )); + + let mut digest = catalog_bytes(GENERATION_ONE_HEX)?; + let digest_offset = digest.len().checked_sub(32).ok_or("catalog too short")?; + *digest + .get_mut(digest_offset) + .ok_or("catalog lacks digest")? ^= 1; + assert!(matches!( + require_error( + ChecksummedCatalog::decode(&digest), + "corrupt catalog digest was admitted" + )?, + CatalogDecodeError::DigestMismatch { .. } + )); + Ok(()) +} diff --git a/tests/catalog/mutation_support.rs b/tests/catalog/mutation_support.rs new file mode 100644 index 0000000..456efaf --- /dev/null +++ b/tests/catalog/mutation_support.rs @@ -0,0 +1,87 @@ +//! Integrity-valid catalog mutation support. +#![allow( + clippy::redundant_pub_crate, + reason = "sibling law modules share these private test fixtures" +)] + +use std::error::Error; + +use keep::{CatalogDecodeError, ChecksummedCatalog}; + +use super::{GENERATION_ONE_HEX, catalog_bytes, format_oracle}; +use crate::support::require_error; + +pub(crate) fn assert_byte_refusal( + offset: usize, + value: u8, + predicate: impl FnOnce(CatalogDecodeError) -> bool, +) -> Result<(), Box> { + let mut encoded = catalog_bytes(GENERATION_ONE_HEX)?; + *encoded + .get_mut(offset) + .ok_or("catalog fixture lacks mutation offset")? = value; + assert_catalog_refusal(&mut encoded, predicate) +} + +pub(crate) fn assert_u16_refusal( + offset: usize, + value: u16, + predicate: impl FnOnce(CatalogDecodeError) -> bool, +) -> Result<(), Box> { + let mut encoded = catalog_bytes(GENERATION_ONE_HEX)?; + replace_range(&mut encoded, offset, &value.to_be_bytes())?; + assert_catalog_refusal(&mut encoded, predicate) +} + +pub(crate) fn assert_u64_refusal( + offset: usize, + value: u64, + predicate: impl FnOnce(CatalogDecodeError) -> bool, +) -> Result<(), Box> { + let mut encoded = catalog_bytes(GENERATION_ONE_HEX)?; + replace_range(&mut encoded, offset, &value.to_be_bytes())?; + assert_catalog_refusal(&mut encoded, predicate) +} + +pub(crate) fn assert_catalog_refusal( + encoded: &mut [u8], + predicate: impl FnOnce(CatalogDecodeError) -> bool, +) -> Result<(), Box> { + format_oracle::seal(encoded)?; + let error = require_error( + ChecksummedCatalog::decode(encoded), + "mutated catalog was admitted", + )?; + assert!(predicate(error), "unexpected refusal: {error:?}"); + Ok(()) +} + +pub(crate) fn replace_range( + target: &mut [u8], + offset: usize, + replacement: &[u8], +) -> Result<(), Box> { + let end = offset + .checked_add(replacement.len()) + .ok_or("test mutation offset overflow")?; + target + .get_mut(offset..end) + .ok_or("catalog fixture lacks mutation field")? + .copy_from_slice(replacement); + Ok(()) +} + +pub(crate) fn zero_range( + target: &mut [u8], + offset: usize, + length: usize, +) -> Result<(), Box> { + let end = offset + .checked_add(length) + .ok_or("test zeroing offset overflow")?; + target + .get_mut(offset..end) + .ok_or("catalog fixture lacks zeroing field")? + .fill(0); + Ok(()) +} diff --git a/tests/catalog/ordering_laws.rs b/tests/catalog/ordering_laws.rs new file mode 100644 index 0000000..66863e5 --- /dev/null +++ b/tests/catalog/ordering_laws.rs @@ -0,0 +1,52 @@ +//! Catalog strict logical-identity ordering laws. + +use std::error::Error; + +use keep::CatalogDecodeError; + +use super::{BUNDLE_HEX, ENTRY_LENGTH, FIRST_ENTRY_OFFSET, catalog_bytes, mutation_support}; + +#[test] +fn catalog_refuses_duplicate_logical_identities() -> Result<(), Box> { + let mut encoded = catalog_bytes(BUNDLE_HEX)?; + let second_offset = FIRST_ENTRY_OFFSET + .checked_add(ENTRY_LENGTH) + .ok_or("second-entry offset overflow")?; + let first = entry(&encoded, FIRST_ENTRY_OFFSET)?.to_vec(); + mutation_support::replace_range(&mut encoded, second_offset, &first)?; + mutation_support::assert_catalog_refusal(&mut encoded, |error| { + error + == CatalogDecodeError::DuplicateIdentity { + first_index: 0, + duplicate_index: 1, + } + }) +} + +#[test] +fn catalog_refuses_out_of_order_logical_identities() -> Result<(), Box> { + let mut encoded = catalog_bytes(BUNDLE_HEX)?; + let second_offset = FIRST_ENTRY_OFFSET + .checked_add(ENTRY_LENGTH) + .ok_or("second-entry offset overflow")?; + let first = entry(&encoded, FIRST_ENTRY_OFFSET)?.to_vec(); + let second = entry(&encoded, second_offset)?.to_vec(); + mutation_support::replace_range(&mut encoded, FIRST_ENTRY_OFFSET, &second)?; + mutation_support::replace_range(&mut encoded, second_offset, &first)?; + mutation_support::assert_catalog_refusal(&mut encoded, |error| { + error + == CatalogDecodeError::IdentityOrder { + previous_index: 0, + observed_index: 1, + } + }) +} + +fn entry(encoded: &[u8], offset: usize) -> Result<&[u8], Box> { + let end = offset + .checked_add(ENTRY_LENGTH) + .ok_or("entry offset overflow")?; + encoded + .get(offset..end) + .ok_or_else(|| "catalog fixture lacks complete entry".into()) +} From 61e13de2d2269e7639549e8b793305f9127d0059 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 23:53:58 -0700 Subject: [PATCH 05/31] Bind catalogs to admitted records --- src/adapters/admitted_catalog.rs | 58 ++++++++ src/adapters/admitted_segment.rs | 5 + src/adapters/catalog_admission.rs | 138 ++++++++++++++++++ src/adapters/catalog_admission_error.rs | 81 ++++++++++ .../catalog_admission_error_display.rs | 64 ++++++++ src/adapters/catalog_allocation_phase.rs | 21 +++ src/adapters/catalog_entries.rs | 57 ++++++++ src/adapters/catalog_entry_decoder.rs | 15 +- src/adapters/catalog_entry_sequence.rs | 6 +- src/adapters/catalog_record_binding.rs | 26 ++++ src/adapters/checksummed_catalog.rs | 28 +++- src/adapters/decoded_catalog_entry.rs | 50 +++++++ src/adapters/mod.rs | 15 ++ src/lib.rs | 13 +- tests/catalog_locations.rs | 129 ++++++++++++++++ tests/catalog_locations/refusal_laws.rs | 115 +++++++++++++++ 16 files changed, 808 insertions(+), 13 deletions(-) create mode 100644 src/adapters/admitted_catalog.rs create mode 100644 src/adapters/catalog_admission.rs create mode 100644 src/adapters/catalog_admission_error.rs create mode 100644 src/adapters/catalog_admission_error_display.rs create mode 100644 src/adapters/catalog_allocation_phase.rs create mode 100644 src/adapters/catalog_entries.rs create mode 100644 src/adapters/catalog_record_binding.rs create mode 100644 src/adapters/decoded_catalog_entry.rs create mode 100644 tests/catalog_locations.rs create mode 100644 tests/catalog_locations/refusal_laws.rs diff --git a/src/adapters/admitted_catalog.rs b/src/adapters/admitted_catalog.rs new file mode 100644 index 0000000..0e6a918 --- /dev/null +++ b/src/adapters/admitted_catalog.rs @@ -0,0 +1,58 @@ +//! Catalog whose logical records are bound to admitted segment bytes. + +use super::{ + AdmittedSegmentRecord, CatalogRecordBinding, ChecksummedCatalog, SegmentRecordIdentity, +}; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Immutable catalog snapshot over exact content-admitted segment records. +/// +/// Lookups expose logical identities and verified record bytes. Physical +/// segment names, offsets, and lengths remain representation details. +#[must_use] +#[derive(Debug)] +pub struct AdmittedCatalog<'catalog, 'records> { + catalog: ChecksummedCatalog<'catalog>, + records: Vec>, +} + +impl<'catalog, 'records> AdmittedCatalog<'catalog, 'records> { + /// Returns the immutable catalog generation. + pub const fn generation(&self) -> CatalogGeneration { + self.catalog.generation() + } + + /// Returns the verified physical catalog digest. + pub const fn digest(&self) -> CatalogDigest { + self.catalog.digest() + } + + /// Returns the exact number of logical record bindings. + #[must_use] + pub const fn record_count(&self) -> u64 { + self.catalog.entry_count() + } + + /// Looks up one logical identity without exposing its physical location. + #[must_use] + pub fn record( + &self, + identity: SegmentRecordIdentity, + ) -> Option> { + let index = self + .records + .binary_search_by_key(&identity, |binding| binding.identity()) + .ok()?; + self.records + .get(index) + .copied() + .map(CatalogRecordBinding::record) + } + + pub(super) const fn from_verified_parts( + catalog: ChecksummedCatalog<'catalog>, + records: Vec>, + ) -> Self { + Self { catalog, records } + } +} diff --git a/src/adapters/admitted_segment.rs b/src/adapters/admitted_segment.rs index 714e447..f800819 100644 --- a/src/adapters/admitted_segment.rs +++ b/src/adapters/admitted_segment.rs @@ -1,5 +1,6 @@ //! Structurally and logically admitted borrowed immutable segment. +use super::segment_record_cursor::SegmentRecordCursor; use super::{ SegmentDigest, SegmentReadError, SegmentReadPolicy, SegmentRecords, SegmentSeal, segment_reader, }; @@ -56,6 +57,10 @@ impl<'a> AdmittedSegment<'a> { SegmentRecords::new(self.records, self.seal.record_count(), self.policy) } + pub(super) const fn record_cursor(&self) -> SegmentRecordCursor<'a> { + SegmentRecordCursor::new(self.records, self.seal.record_count(), self.policy) + } + pub(super) const fn admitted( encoded: &'a [u8], records: &'a [u8], diff --git a/src/adapters/catalog_admission.rs b/src/adapters/catalog_admission.rs new file mode 100644 index 0000000..4fad970 --- /dev/null +++ b/src/adapters/catalog_admission.rs @@ -0,0 +1,138 @@ +//! Binding of catalog-local coordinates to exact admitted segment records. + +use super::{ + AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, CatalogAdmissionError, + CatalogAllocationPhase, CatalogRecordBinding, ChecksummedCatalog, DecodedCatalogEntry, + SegmentDigest, +}; + +pub(super) fn admit<'catalog, 'records>( + catalog: ChecksummedCatalog<'catalog>, + segments: &[AdmittedSegment<'records>], +) -> Result, CatalogAdmissionError> { + let requested = usize::try_from(catalog.entry_count()).map_err(|_source| { + CatalogAdmissionError::EntryCountHostWidth { + observed: catalog.entry_count(), + } + })?; + if segments.len() > requested { + return Err(CatalogAdmissionError::SegmentCountOutOfBounds { + maximum: catalog.entry_count(), + observed: segments.len(), + }); + } + let segment_index = index_segments(segments)?; + let mut bindings = Vec::new(); + bindings + .try_reserve_exact(requested) + .map_err(|source| CatalogAdmissionError::Allocation { + phase: CatalogAllocationPhase::RecordBindings, + requested, + source, + })?; + let entries = catalog + .entries() + .map_err(|source| CatalogAdmissionError::Catalog { source })?; + for entry in entries { + let entry = entry.map_err(|source| CatalogAdmissionError::Catalog { source })?; + let segment = find_segment(&segment_index, entry.segment_digest())?; + let record = locate_record(segment, entry)?; + validate_record(entry, record)?; + bindings.push(CatalogRecordBinding::new(entry.identity(), record)); + } + Ok(AdmittedCatalog::from_verified_parts(catalog, bindings)) +} + +fn index_segments<'slice, 'records>( + segments: &'slice [AdmittedSegment<'records>], +) -> Result>, CatalogAdmissionError> { + let requested = segments.len(); + let mut indexed = Vec::new(); + indexed + .try_reserve_exact(requested) + .map_err(|source| CatalogAdmissionError::Allocation { + phase: CatalogAllocationPhase::SegmentIndex, + requested, + source, + })?; + indexed.extend(segments); + indexed.sort_unstable_by_key(|segment| segment.digest()); + for pair in indexed.windows(2) { + let [first, second] = pair else { + continue; + }; + if first.digest() == second.digest() { + return Err(CatalogAdmissionError::DuplicateSegment { + digest: first.digest(), + }); + } + } + Ok(indexed) +} + +fn find_segment<'slice, 'records>( + segments: &'slice [&AdmittedSegment<'records>], + digest: SegmentDigest, +) -> Result<&'slice AdmittedSegment<'records>, CatalogAdmissionError> { + let index = segments + .binary_search_by_key(&digest, |segment| segment.digest()) + .map_err(|_source| CatalogAdmissionError::MissingSegment { digest })?; + segments + .get(index) + .copied() + .ok_or(CatalogAdmissionError::MissingSegment { digest }) +} + +fn locate_record<'records>( + segment: &AdmittedSegment<'records>, + entry: DecodedCatalogEntry, +) -> Result, CatalogAdmissionError> { + let digest = segment.digest(); + let mut cursor = segment.record_cursor(); + let mut found = None; + while let Some(located) = + cursor + .next_record() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })? + { + if located.offset == entry.record_offset() + && located.record.header().record_length() == entry.record_length() + { + found = Some(located.record); + } + } + cursor + .finish() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })?; + found.ok_or_else(|| CatalogAdmissionError::LocationNotTopLevel { + identity: entry.identity(), + segment_digest: entry.segment_digest(), + record_offset: entry.record_offset(), + record_length: entry.record_length().get(), + }) +} + +fn validate_record( + entry: DecodedCatalogEntry, + record: AdmittedSegmentRecord<'_>, +) -> Result<(), CatalogAdmissionError> { + if record.identity() != entry.identity() { + return Err(CatalogAdmissionError::RecordIdentityMismatch { + expected: entry.identity(), + observed: record.identity(), + }); + } + if record.checksum() != entry.checksum() { + return Err(CatalogAdmissionError::RecordChecksumMismatch { + expected: entry.checksum(), + observed: record.checksum(), + }); + } + Ok(()) +} diff --git a/src/adapters/catalog_admission_error.rs b/src/adapters/catalog_admission_error.rs new file mode 100644 index 0000000..52cbd42 --- /dev/null +++ b/src/adapters/catalog_admission_error.rs @@ -0,0 +1,81 @@ +//! Catalog-to-segment admission failures. + +use std::collections::TryReserveError; + +use super::{ + CatalogAllocationPhase, CatalogDecodeError, SegmentDigest, SegmentReadError, + SegmentRecordChecksum, SegmentRecordIdentity, +}; + +/// Failure to bind a checksummed catalog to exact admitted segment records. +#[derive(Debug)] +pub enum CatalogAdmissionError { + /// Revalidating an immutable catalog entry failed. + Catalog { + /// Exact nested catalog failure. + source: CatalogDecodeError, + }, + /// The bounded catalog count cannot fit the host allocation width. + EntryCountHostWidth { + /// Verified catalog entry count. + observed: u64, + }, + /// Caller input supplied more segments than the catalog could reference. + SegmentCountOutOfBounds { + /// Largest useful segment count. + maximum: u64, + /// Caller-supplied segment count. + observed: usize, + }, + /// A bounded segment-index or record-binding allocation failed. + Allocation { + /// Semantic allocation phase. + phase: CatalogAllocationPhase, + /// Exact requested element capacity. + requested: usize, + /// Allocator refusal. + source: TryReserveError, + }, + /// Caller input repeated one physical segment digest. + DuplicateSegment { + /// Repeated physical segment coordinate. + digest: SegmentDigest, + }, + /// The catalog named a segment absent from caller input. + MissingSegment { + /// Required physical segment coordinate. + digest: SegmentDigest, + }, + /// Revalidating an admitted segment's immutable records failed. + Segment { + /// Physical segment being scanned. + digest: SegmentDigest, + /// Exact nested segment failure. + source: Box, + }, + /// A physical coordinate was not one complete top-level record span. + LocationNotTopLevel { + /// Logical catalog key. + identity: SegmentRecordIdentity, + /// Physical segment coordinate. + segment_digest: SegmentDigest, + /// Declared absolute record offset. + record_offset: u64, + /// Declared complete-record length. + record_length: u64, + }, + /// The selected record carried a different logical identity. + RecordIdentityMismatch { + /// Identity declared by the catalog. + expected: SegmentRecordIdentity, + /// Identity verified from the selected record. + observed: SegmentRecordIdentity, + }, + /// The selected record carried a different checksum. + RecordChecksumMismatch { + /// Checksum declared by the catalog. + expected: SegmentRecordChecksum, + /// Checksum verified from the selected record. + observed: SegmentRecordChecksum, + }, +} diff --git a/src/adapters/catalog_admission_error_display.rs b/src/adapters/catalog_admission_error_display.rs new file mode 100644 index 0000000..7370723 --- /dev/null +++ b/src/adapters/catalog_admission_error_display.rs @@ -0,0 +1,64 @@ +//! Human-readable catalog-admission diagnostics. + +use std::error::Error; +use std::fmt; + +use super::CatalogAdmissionError; + +impl fmt::Display for CatalogAdmissionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Catalog { source } => { + write!(formatter, "catalog revalidation failed: {source}") + } + Self::EntryCountHostWidth { observed } => { + write!( + formatter, + "catalog entry count {observed} exceeds host width" + ) + } + Self::SegmentCountOutOfBounds { maximum, observed } => write!( + formatter, + "catalog admits at most {maximum} segments, observed {observed}" + ), + Self::Allocation { + phase, requested, .. + } => write!( + formatter, + "catalog {phase} allocation failed for {requested} elements" + ), + Self::DuplicateSegment { .. } => { + formatter.write_str("duplicate admitted segment digest") + } + Self::MissingSegment { .. } => formatter.write_str("catalog segment is missing"), + Self::Segment { source, .. } => { + write!(formatter, "catalog segment revalidation failed: {source}") + } + Self::LocationNotTopLevel { + record_offset, + record_length, + .. + } => write!( + formatter, + "catalog location {record_offset}+{record_length} is not a top-level record" + ), + Self::RecordIdentityMismatch { .. } => { + formatter.write_str("catalog and record identities disagree") + } + Self::RecordChecksumMismatch { .. } => { + formatter.write_str("catalog and record checksums disagree") + } + } + } +} + +impl Error for CatalogAdmissionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Catalog { source } => Some(source), + Self::Allocation { source, .. } => Some(source), + Self::Segment { source, .. } => Some(source.as_ref()), + _ => None, + } + } +} diff --git a/src/adapters/catalog_allocation_phase.rs b/src/adapters/catalog_allocation_phase.rs new file mode 100644 index 0000000..a8b79e3 --- /dev/null +++ b/src/adapters/catalog_allocation_phase.rs @@ -0,0 +1,21 @@ +//! Bounded catalog-admission allocation phases. + +use std::fmt; + +/// Bounded temporary or retained allocation attempted by catalog admission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogAllocationPhase { + /// Sorted borrowed index over caller-supplied admitted segments. + SegmentIndex, + /// Logical-identity bindings retained by the admitted catalog. + RecordBindings, +} + +impl fmt::Display for CatalogAllocationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SegmentIndex => formatter.write_str("segment index"), + Self::RecordBindings => formatter.write_str("record bindings"), + } + } +} diff --git a/src/adapters/catalog_entries.rs b/src/adapters/catalog_entries.rs new file mode 100644 index 0000000..c347ec7 --- /dev/null +++ b/src/adapters/catalog_entries.rs @@ -0,0 +1,57 @@ +//! Revalidating iterator over one checksummed catalog's fixed-width entries. + +use std::slice::ChunksExact; + +use super::{ + CatalogDecodeError, DecodedCatalogEntry, catalog_entry_decoder, catalog_header_decoder, +}; + +pub(super) struct CatalogEntries<'a> { + chunks: ChunksExact<'a, u8>, + next_index: u64, + entry_count: u64, +} + +impl<'a> CatalogEntries<'a> { + pub(super) fn new(encoded: &'a [u8], entry_count: u64) -> Result { + let entries_end = encoded + .len() + .checked_sub(catalog_header_decoder::TRAILER_LENGTH) + .ok_or(CatalogDecodeError::MinimumLength { + minimum: catalog_header_decoder::MINIMUM_LENGTH, + observed: encoded.len(), + })?; + let entries = encoded + .get(catalog_header_decoder::HEADER_LENGTH_BYTES..entries_end) + .ok_or(CatalogDecodeError::MinimumLength { + minimum: catalog_header_decoder::MINIMUM_LENGTH, + observed: encoded.len(), + })?; + Ok(Self { + chunks: entries.chunks_exact(catalog_entry_decoder::ENCODED_LENGTH), + next_index: 0, + entry_count, + }) + } +} + +impl Iterator for CatalogEntries<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + let encoded = self.chunks.next()?; + let index = self.next_index; + self.next_index = match index.checked_add(1) { + Some(next) => next, + None => { + return Some(Err(CatalogDecodeError::LengthArithmetic { + entry_count: self.entry_count, + })); + } + }; + Some( + catalog_entry_decoder::decode(encoded) + .map_err(|source| CatalogDecodeError::Entry { index, source }), + ) + } +} diff --git a/src/adapters/catalog_entry_decoder.rs b/src/adapters/catalog_entry_decoder.rs index 814ea97..3f0f357 100644 --- a/src/adapters/catalog_entry_decoder.rs +++ b/src/adapters/catalog_entry_decoder.rs @@ -1,7 +1,8 @@ //! Canonical fixed-width catalog-entry decoder. use super::{ - CatalogEntryDecodeError, SegmentRecordIdentity, + CatalogEntryDecodeError, DecodedCatalogEntry, SegmentDigest, SegmentRecordChecksum, + SegmentRecordIdentity, SegmentRecordLength, catalog_entry_fields::{self, read_array, read_u8, read_u16, read_u64}, }; use crate::{ChunkId, ChunkLength, LayoutId}; @@ -16,7 +17,7 @@ const SEGMENT_HEADER_LENGTH: u64 = 64; const RECORD_FRAMING_LENGTH: u64 = 144; const MAXIMUM_RECORD_PAYLOAD_LENGTH: u64 = 67_108_864; -pub(super) fn decode(encoded: &[u8]) -> Result { +pub(super) fn decode(encoded: &[u8]) -> Result { if encoded.len() != ENCODED_LENGTH { return Err(CatalogEntryDecodeError::WrongLength { expected: ENCODED_LENGTH, @@ -33,9 +34,11 @@ pub(super) fn decode(encoded: &[u8]) -> Result Result Result<(), CatalogDe { let index = u64::try_from(host_index) .map_err(|_source| CatalogDecodeError::LengthArithmetic { entry_count })?; - let identity = catalog_entry_decoder::decode(entry) + let decoded = catalog_entry_decoder::decode(entry) .map_err(|source| CatalogDecodeError::Entry { index, source })?; - validate_order(previous, index, identity)?; - previous = Some((index, identity)); + validate_order(previous, index, decoded.identity())?; + previous = Some((index, decoded.identity())); } Ok(()) } diff --git a/src/adapters/catalog_record_binding.rs b/src/adapters/catalog_record_binding.rs new file mode 100644 index 0000000..a1fd8ac --- /dev/null +++ b/src/adapters/catalog_record_binding.rs @@ -0,0 +1,26 @@ +//! Internal logical identity to admitted-record binding. + +use super::{AdmittedSegmentRecord, SegmentRecordIdentity}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct CatalogRecordBinding<'a> { + identity: SegmentRecordIdentity, + record: AdmittedSegmentRecord<'a>, +} + +impl<'a> CatalogRecordBinding<'a> { + pub(super) const fn new( + identity: SegmentRecordIdentity, + record: AdmittedSegmentRecord<'a>, + ) -> Self { + Self { identity, record } + } + + pub(super) const fn identity(self) -> SegmentRecordIdentity { + self.identity + } + + pub(super) const fn record(self) -> AdmittedSegmentRecord<'a> { + self.record + } +} diff --git a/src/adapters/checksummed_catalog.rs b/src/adapters/checksummed_catalog.rs index 2b7494a..898fd52 100644 --- a/src/adapters/checksummed_catalog.rs +++ b/src/adapters/checksummed_catalog.rs @@ -1,6 +1,9 @@ //! Canonically framed, checksum- and digest-verified borrowed catalog. -use super::{CatalogDecodeError, catalog_decoder}; +use super::{ + AdmittedCatalog, AdmittedSegment, CatalogAdmissionError, CatalogDecodeError, CatalogEntries, + catalog_admission, catalog_decoder, +}; use crate::{CatalogDigest, CatalogGeneration}; /// Borrowed catalog bytes with canonical framing, ordering, and integrity proof. @@ -31,6 +34,25 @@ impl<'a> ChecksummedCatalog<'a> { catalog_decoder::decode(encoded) } + /// Binds every logical entry to one exact top-level admitted segment record. + /// + /// This operation performs no I/O. It temporarily allocates one sorted + /// borrowed segment index and retains one logical record binding per entry. + /// Both allocations are bounded by caller input or the verified catalog + /// entry count. + /// + /// # Errors + /// + /// Returns [`CatalogAdmissionError`] for allocation refusal, duplicate or + /// missing segments, failed immutable revalidation, interior locations, or + /// disagreement between catalog fields and the selected record. + pub fn admit<'records>( + self, + segments: &[AdmittedSegment<'records>], + ) -> Result, CatalogAdmissionError> { + catalog_admission::admit(self, segments) + } + /// Returns the exact borrowed canonical bytes. #[must_use] pub const fn encoded(self) -> &'a [u8] { @@ -59,6 +81,10 @@ impl<'a> ChecksummedCatalog<'a> { self.digest } + pub(super) fn entries(self) -> Result, CatalogDecodeError> { + CatalogEntries::new(self.encoded, self.entry_count) + } + pub(super) const fn from_verified_parts( encoded: &'a [u8], generation: CatalogGeneration, diff --git a/src/adapters/decoded_catalog_entry.rs b/src/adapters/decoded_catalog_entry.rs new file mode 100644 index 0000000..ad10bc7 --- /dev/null +++ b/src/adapters/decoded_catalog_entry.rs @@ -0,0 +1,50 @@ +//! Internal semantic catalog entry with unadmitted physical coordinates. + +use super::{SegmentDigest, SegmentRecordChecksum, SegmentRecordIdentity, SegmentRecordLength}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct DecodedCatalogEntry { + identity: SegmentRecordIdentity, + segment_digest: SegmentDigest, + record_offset: u64, + record_length: SegmentRecordLength, + checksum: SegmentRecordChecksum, +} + +impl DecodedCatalogEntry { + pub(super) const fn new( + identity: SegmentRecordIdentity, + segment_digest: SegmentDigest, + record_offset: u64, + record_length: SegmentRecordLength, + checksum: SegmentRecordChecksum, + ) -> Self { + Self { + identity, + segment_digest, + record_offset, + record_length, + checksum, + } + } + + pub(super) const fn identity(self) -> SegmentRecordIdentity { + self.identity + } + + pub(super) const fn segment_digest(self) -> SegmentDigest { + self.segment_digest + } + + pub(super) const fn record_offset(self) -> u64 { + self.record_offset + } + + pub(super) const fn record_length(self) -> SegmentRecordLength { + self.record_length + } + + pub(super) const fn checksum(self) -> SegmentRecordChecksum { + self.checksum + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 3502ba7..d421e9a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -5,15 +5,21 @@ //! ingress and egress. It does not own identity calculation, logical layout //! policy, physical location, namespace publication, recovery, or retention. +mod admitted_catalog; mod admitted_segment; mod admitted_segment_record; mod blob_id_binary; mod blob_id_binary_error; mod blob_id_text; mod blob_id_text_error; +mod catalog_admission; +mod catalog_admission_error; +mod catalog_admission_error_display; +mod catalog_allocation_phase; mod catalog_decode_error; mod catalog_decode_error_display; mod catalog_decoder; +mod catalog_entries; mod catalog_entry_decode_error; mod catalog_entry_decode_error_display; mod catalog_entry_decoder; @@ -21,9 +27,11 @@ mod catalog_entry_fields; mod catalog_entry_sequence; mod catalog_header_decoder; mod catalog_integrity; +mod catalog_record_binding; mod checksummed_catalog; mod checksummed_publication_head; mod checksummed_segment_record; +mod decoded_catalog_entry; mod filesystem_segment_stage; mod framed_blake3; mod layout_decode_error; @@ -101,10 +109,13 @@ mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; +pub use admitted_catalog::AdmittedCatalog; pub use admitted_segment::AdmittedSegment; pub use admitted_segment_record::AdmittedSegmentRecord; pub use blob_id_binary_error::BlobIdBinaryParseError; pub use blob_id_text_error::BlobIdTextParseError; +pub use catalog_admission_error::CatalogAdmissionError; +pub use catalog_allocation_phase::CatalogAllocationPhase; pub use catalog_decode_error::CatalogDecodeError; pub use catalog_entry_decode_error::CatalogEntryDecodeError; pub use checksummed_catalog::ChecksummedCatalog; @@ -142,3 +153,7 @@ pub use segment_write_error::SegmentWriteError; pub use segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use staged_segment::StagedSegment; pub use storage_profile_id_text_error::StorageProfileIdParseError; + +use catalog_entries::CatalogEntries; +use catalog_record_binding::CatalogRecordBinding; +use decoded_catalog_entry::DecodedCatalogEntry; diff --git a/src/lib.rs b/src/lib.rs index 6e22368..90bac52 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,12 +20,13 @@ mod profile; mod reference; pub use adapters::{ - AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalLayoutRecord, CatalogDecodeError, CatalogEntryDecodeError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemSegmentStage, - LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, - LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, + AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, + BlobIdTextParseError, CanonicalLayoutRecord, CatalogAdmissionError, CatalogAllocationPhase, + CatalogDecodeError, CatalogEntryDecodeError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, + SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, diff --git a/tests/catalog_locations.rs b/tests/catalog_locations.rs new file mode 100644 index 0000000..8160e68 --- /dev/null +++ b/tests/catalog_locations.rs @@ -0,0 +1,129 @@ +//! Catalog-to-segment top-level record binding laws. + +#[path = "catalog/format_oracle.rs"] +mod format_oracle; +#[path = "catalog_locations/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, CatalogAdmissionError, ChecksummedCatalog, ChunkId, LayoutEntryLimit, + SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, +}; +use support::{decode_hex, require_error}; + +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const RECORD_OFFSET_FIELD: usize = 128 + 96; +const ENTRY_IDENTITY_DIGEST_FIELD: usize = 128 + 8; +const ENTRY_SEGMENT_DIGEST_FIELD: usize = 128 + 64; +const ENTRY_CHECKSUM_FIELD: usize = 128 + 120; + +#[test] +fn admitted_catalog_resolves_logical_records_without_exposing_locations() +-> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let admitted = catalog.admit(&[segment])?; + let identity = SegmentRecordIdentity::Chunk(ChunkId::hash_bytes(&[0])?); + let record = admitted + .record(identity) + .ok_or("admitted catalog omitted its logical record")?; + + assert_eq!(admitted.generation().get(), 1); + assert_eq!(admitted.record_count(), 1); + assert_eq!(record.identity(), identity); + assert_eq!(record.payload(), [0]); + Ok(()) +} + +#[test] +fn admitted_bundle_resolves_every_catalog_identity() -> Result<(), Box> { + let catalog_bytes = fixture(BUNDLE_CATALOG_HEX)?; + let segment_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let expected = segment + .records() + .map(|record| record.map(keep::AdmittedSegmentRecord::identity)) + .collect::, _>>()?; + let admitted = catalog.admit(&[segment])?; + + assert_eq!(admitted.record_count(), 2); + for identity in expected { + assert_eq!( + admitted + .record(identity) + .ok_or("bundle catalog omitted a logical record")? + .identity(), + identity + ); + } + Ok(()) +} + +#[test] +fn catalog_requires_the_exact_named_admitted_segment() -> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let error = require_error( + catalog.admit(&[]), + "catalog without its named segment was admitted", + )?; + + assert!(matches!( + error, + CatalogAdmissionError::MissingSegment { .. } + )); + Ok(()) +} + +#[test] +fn catalog_location_must_equal_one_discovered_top_level_record_span() -> Result<(), Box> +{ + let mut catalog_bytes = fixture(CATALOG_HEX)?; + replace_u64(&mut catalog_bytes, RECORD_OFFSET_FIELD, 65)?; + format_oracle::seal(&mut catalog_bytes)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let error = require_error( + catalog.admit(&[segment]), + "interior record location was admitted", + )?; + + assert!(matches!( + error, + CatalogAdmissionError::LocationNotTopLevel { + record_offset: 65, + record_length: 145, + .. + } + )); + Ok(()) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn replace_u64(target: &mut [u8], offset: usize, value: u64) -> Result<(), Box> { + let end = offset.checked_add(8).ok_or("test offset overflow")?; + target + .get_mut(offset..end) + .ok_or("catalog fixture lacks u64 field")? + .copy_from_slice(&value.to_be_bytes()); + Ok(()) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/catalog_locations/refusal_laws.rs b/tests/catalog_locations/refusal_laws.rs new file mode 100644 index 0000000..8fba634 --- /dev/null +++ b/tests/catalog_locations/refusal_laws.rs @@ -0,0 +1,115 @@ +//! Catalog location-binding disagreement laws. + +use std::error::Error; + +use keep::{AdmittedSegment, CatalogAdmissionError, ChecksummedCatalog}; + +use super::{ + BUNDLE_CATALOG_HEX, BUNDLE_SEGMENT_HEX, CATALOG_HEX, ENTRY_CHECKSUM_FIELD, + ENTRY_IDENTITY_DIGEST_FIELD, ENTRY_SEGMENT_DIGEST_FIELD, SEGMENT_HEX, fixture, format_oracle, + maximum_policy, +}; +use crate::support::require_error; + +#[test] +fn catalog_identity_must_equal_the_selected_record_identity() -> Result<(), Box> { + let mut encoded = fixture(CATALOG_HEX)?; + replace_byte(&mut encoded, ENTRY_IDENTITY_DIGEST_FIELD)?; + format_oracle::seal(&mut encoded)?; + let catalog = ChecksummedCatalog::decode(&encoded)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let error = require_error( + catalog.admit(&[segment]), + "catalog identity disagreement was admitted", + )?; + + assert!(matches!( + error, + CatalogAdmissionError::RecordIdentityMismatch { .. } + )); + Ok(()) +} + +#[test] +fn catalog_checksum_must_equal_the_selected_record_checksum() -> Result<(), Box> { + let mut encoded = fixture(CATALOG_HEX)?; + replace_byte(&mut encoded, ENTRY_CHECKSUM_FIELD)?; + format_oracle::seal(&mut encoded)?; + let catalog = ChecksummedCatalog::decode(&encoded)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let error = require_error( + catalog.admit(&[segment]), + "catalog checksum disagreement was admitted", + )?; + + assert!(matches!( + error, + CatalogAdmissionError::RecordChecksumMismatch { .. } + )); + Ok(()) +} + +#[test] +fn catalog_segment_digest_selects_the_exact_physical_segment() -> Result<(), Box> { + let mut encoded = fixture(CATALOG_HEX)?; + replace_byte(&mut encoded, ENTRY_SEGMENT_DIGEST_FIELD)?; + format_oracle::seal(&mut encoded)?; + let catalog = ChecksummedCatalog::decode(&encoded)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let error = require_error( + catalog.admit(&[segment]), + "wrong physical segment digest was admitted", + )?; + + assert!(matches!( + error, + CatalogAdmissionError::MissingSegment { .. } + )); + Ok(()) +} + +#[test] +fn segment_input_is_bounded_and_duplicate_free() -> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let first = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let second = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let error = require_error( + catalog.admit(&[first, second]), + "excess segment input was admitted", + )?; + assert!(matches!( + error, + CatalogAdmissionError::SegmentCountOutOfBounds { + maximum: 1, + observed: 2, + } + )); + + let bundle_catalog_bytes = fixture(BUNDLE_CATALOG_HEX)?; + let bundle_segment_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let bundle_catalog = ChecksummedCatalog::decode(&bundle_catalog_bytes)?; + let first = AdmittedSegment::decode(&bundle_segment_bytes, maximum_policy())?; + let second = AdmittedSegment::decode(&bundle_segment_bytes, maximum_policy())?; + let error = require_error( + bundle_catalog.admit(&[first, second]), + "duplicate physical segment input was admitted", + )?; + assert!(matches!( + error, + CatalogAdmissionError::DuplicateSegment { .. } + )); + Ok(()) +} + +fn replace_byte(target: &mut [u8], offset: usize) -> Result<(), Box> { + let byte = target + .get_mut(offset) + .ok_or("catalog fixture lacks mutation byte")?; + *byte ^= 1; + Ok(()) +} From 90356fc9c25bf60b006a9278c7f981fedc93e159 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 23:57:24 -0700 Subject: [PATCH 06/31] Enforce exact catalog successors --- src/adapters/admitted_catalog.rs | 23 +++- src/adapters/catalog_successor.rs | 32 ++++++ src/adapters/catalog_transition.rs | 23 ++++ src/adapters/catalog_transition_error.rs | 58 ++++++++++ src/adapters/mod.rs | 5 + src/lib.rs | 20 ++-- tests/catalog_transition.rs | 136 +++++++++++++++++++++++ 7 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 src/adapters/catalog_successor.rs create mode 100644 src/adapters/catalog_transition.rs create mode 100644 src/adapters/catalog_transition_error.rs create mode 100644 tests/catalog_transition.rs diff --git a/src/adapters/admitted_catalog.rs b/src/adapters/admitted_catalog.rs index 0e6a918..883f1c9 100644 --- a/src/adapters/admitted_catalog.rs +++ b/src/adapters/admitted_catalog.rs @@ -1,7 +1,8 @@ //! Catalog whose logical records are bound to admitted segment bytes. use super::{ - AdmittedSegmentRecord, CatalogRecordBinding, ChecksummedCatalog, SegmentRecordIdentity, + AdmittedSegmentRecord, CatalogRecordBinding, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, SegmentRecordIdentity, catalog_transition, }; use crate::{CatalogDigest, CatalogGeneration}; @@ -27,6 +28,12 @@ impl<'catalog, 'records> AdmittedCatalog<'catalog, 'records> { self.catalog.digest() } + /// Returns the predecessor witness, absent only for generation 1. + #[must_use] + pub const fn previous_catalog_digest(&self) -> Option { + self.catalog.previous_catalog_digest() + } + /// Returns the exact number of logical record bindings. #[must_use] pub const fn record_count(&self) -> u64 { @@ -49,6 +56,20 @@ impl<'catalog, 'records> AdmittedCatalog<'catalog, 'records> { .map(CatalogRecordBinding::record) } + /// Admits a fully verified candidate as this snapshot's exact successor. + /// + /// # Errors + /// + /// Returns [`CatalogTransitionError`] when generation arithmetic is + /// exhausted, the candidate is not exactly one generation later, or its + /// predecessor digest does not equal this catalog's verified digest. + pub fn validate_successor<'next_catalog, 'next_records>( + &self, + candidate: AdmittedCatalog<'next_catalog, 'next_records>, + ) -> Result, CatalogTransitionError> { + catalog_transition::validate(self, candidate) + } + pub(super) const fn from_verified_parts( catalog: ChecksummedCatalog<'catalog>, records: Vec>, diff --git a/src/adapters/catalog_successor.rs b/src/adapters/catalog_successor.rs new file mode 100644 index 0000000..4dd6644 --- /dev/null +++ b/src/adapters/catalog_successor.rs @@ -0,0 +1,32 @@ +//! Exact successor catalog staged for publication. + +use super::AdmittedCatalog; +use crate::CatalogGeneration; + +/// Fully admitted catalog proven to be the exact successor of one snapshot. +#[must_use] +#[derive(Debug)] +pub struct CatalogSuccessor<'catalog, 'records> { + catalog: AdmittedCatalog<'catalog, 'records>, +} + +impl<'catalog, 'records> CatalogSuccessor<'catalog, 'records> { + /// Returns the exact successor generation. + pub const fn generation(&self) -> CatalogGeneration { + self.catalog.generation() + } + + /// Borrows the fully admitted successor catalog. + pub const fn catalog(&self) -> &AdmittedCatalog<'catalog, 'records> { + &self.catalog + } + + /// Consumes the transition proof and returns the admitted catalog. + pub fn into_catalog(self) -> AdmittedCatalog<'catalog, 'records> { + self.catalog + } + + pub(super) const fn new(catalog: AdmittedCatalog<'catalog, 'records>) -> Self { + Self { catalog } + } +} diff --git a/src/adapters/catalog_transition.rs b/src/adapters/catalog_transition.rs new file mode 100644 index 0000000..064178b --- /dev/null +++ b/src/adapters/catalog_transition.rs @@ -0,0 +1,23 @@ +//! Exact generation and predecessor transition admission. + +use super::{AdmittedCatalog, CatalogSuccessor, CatalogTransitionError}; + +pub(super) fn validate<'catalog, 'records>( + current: &AdmittedCatalog<'_, '_>, + candidate: AdmittedCatalog<'catalog, 'records>, +) -> Result, CatalogTransitionError> { + let expected = current + .generation() + .successor() + .map_err(|source| CatalogTransitionError::GenerationExhausted { source })?; + let observed = candidate.generation(); + if observed != expected { + return Err(CatalogTransitionError::Generation { expected, observed }); + } + let expected = current.digest(); + let observed = candidate.previous_catalog_digest(); + if observed != Some(expected) { + return Err(CatalogTransitionError::Predecessor { expected, observed }); + } + Ok(CatalogSuccessor::new(candidate)) +} diff --git a/src/adapters/catalog_transition_error.rs b/src/adapters/catalog_transition_error.rs new file mode 100644 index 0000000..2a2301f --- /dev/null +++ b/src/adapters/catalog_transition_error.rs @@ -0,0 +1,58 @@ +//! Exact catalog-successor transition failures. + +use std::error::Error; +use std::fmt; + +use crate::{CatalogDigest, CatalogGeneration, CatalogGenerationError}; + +/// Failure to admit one exact successor to a current catalog snapshot. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogTransitionError { + /// The current generation has no representable successor. + GenerationExhausted { + /// Exact checked-generation failure. + source: CatalogGenerationError, + }, + /// The candidate generation was not the exact successor. + Generation { + /// Required successor generation. + expected: CatalogGeneration, + /// Candidate generation. + observed: CatalogGeneration, + }, + /// The candidate did not name the current catalog digest. + Predecessor { + /// Required predecessor digest. + expected: CatalogDigest, + /// Candidate predecessor coordinate. + observed: Option, + }, +} + +impl fmt::Display for CatalogTransitionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::GenerationExhausted { source } => { + write!(formatter, "catalog generation is exhausted: {source}") + } + Self::Generation { expected, observed } => write!( + formatter, + "catalog successor generation must be {}, observed {}", + expected.get(), + observed.get() + ), + Self::Predecessor { .. } => { + formatter.write_str("catalog successor predecessor digest mismatch") + } + } + } +} + +impl Error for CatalogTransitionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::GenerationExhausted { source } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index d421e9a..7503f2f 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -28,6 +28,9 @@ mod catalog_entry_sequence; mod catalog_header_decoder; mod catalog_integrity; mod catalog_record_binding; +mod catalog_successor; +mod catalog_transition; +mod catalog_transition_error; mod checksummed_catalog; mod checksummed_publication_head; mod checksummed_segment_record; @@ -118,6 +121,8 @@ pub use catalog_admission_error::CatalogAdmissionError; pub use catalog_allocation_phase::CatalogAllocationPhase; pub use catalog_decode_error::CatalogDecodeError; pub use catalog_entry_decode_error::CatalogEntryDecodeError; +pub use catalog_successor::CatalogSuccessor; +pub use catalog_transition_error::CatalogTransitionError; pub use checksummed_catalog::ChecksummedCatalog; pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; diff --git a/src/lib.rs b/src/lib.rs index 90bac52..9aa6749 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,16 +22,16 @@ mod reference; pub use adapters::{ AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalLayoutRecord, CatalogAdmissionError, CatalogAllocationPhase, - CatalogDecodeError, CatalogEntryDecodeError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, - SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, - SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, - SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, - SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, - SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, - SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + CatalogDecodeError, CatalogEntryDecodeError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, + FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_transition.rs b/tests/catalog_transition.rs new file mode 100644 index 0000000..3a5343a --- /dev/null +++ b/tests/catalog_transition.rs @@ -0,0 +1,136 @@ +//! Exact catalog successor admission laws. + +#[path = "catalog/format_oracle.rs"] +mod format_oracle; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedCatalog, AdmittedSegment, CatalogGenerationError, CatalogTransitionError, + ChecksummedCatalog, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, +}; +use support::{decode_hex, require_error}; + +const GENERATION_ONE_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const GENERATION_TWO_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const GENERATION_FIELD: usize = 24; +const PREDECESSOR_FIELD: usize = 32; + +#[test] +fn exact_successor_preserves_the_admitted_candidate() -> Result<(), Box> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let current_bytes = fixture(GENERATION_ONE_HEX)?; + let candidate_bytes = fixture(GENERATION_TWO_HEX)?; + let current = admitted(¤t_bytes, &segment_bytes)?; + let candidate = admitted(&candidate_bytes, &segment_bytes)?; + let successor = current.validate_successor(candidate)?; + + assert_eq!(successor.generation().get(), 2); + assert_eq!(successor.catalog().record_count(), 1); + Ok(()) +} + +#[test] +fn stale_generation_reports_expected_and_observed_coordinates() -> Result<(), Box> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let current_bytes = fixture(GENERATION_ONE_HEX)?; + let stale_bytes = fixture(GENERATION_ONE_HEX)?; + let current = admitted(¤t_bytes, &segment_bytes)?; + let stale = admitted(&stale_bytes, &segment_bytes)?; + let error = require_error( + current.validate_successor(stale), + "stale generation was admitted as a successor", + )?; + + assert!(matches!( + error, + CatalogTransitionError::Generation { + expected, + observed, + } if expected.get() == 2 && observed.get() == 1 + )); + Ok(()) +} + +#[test] +fn wrong_predecessor_reports_expected_and_observed_digests() -> Result<(), Box> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let current_bytes = fixture(GENERATION_ONE_HEX)?; + let mut candidate_bytes = fixture(GENERATION_TWO_HEX)?; + *candidate_bytes + .get_mut(PREDECESSOR_FIELD) + .ok_or("candidate lacks predecessor digest")? ^= 1; + format_oracle::seal(&mut candidate_bytes)?; + let current = admitted(¤t_bytes, &segment_bytes)?; + let expected = current.digest(); + let candidate = admitted(&candidate_bytes, &segment_bytes)?; + let observed = candidate + .previous_catalog_digest() + .ok_or("mutated candidate omitted its predecessor")?; + let error = require_error( + current.validate_successor(candidate), + "wrong predecessor was admitted as a successor", + )?; + + assert!(matches!( + error, + CatalogTransitionError::Predecessor { + expected: error_expected, + observed: Some(error_observed), + } if error_expected == expected && error_observed == observed + )); + Ok(()) +} + +#[test] +fn maximum_generation_refuses_successor_derivation() -> Result<(), Box> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let mut current_bytes = fixture(GENERATION_TWO_HEX)?; + replace_u64(&mut current_bytes, GENERATION_FIELD, u64::MAX)?; + format_oracle::seal(&mut current_bytes)?; + let candidate_bytes = fixture(GENERATION_TWO_HEX)?; + let current = admitted(¤t_bytes, &segment_bytes)?; + let candidate = admitted(&candidate_bytes, &segment_bytes)?; + let error = require_error( + current.validate_successor(candidate), + "successor was derived after generation exhaustion", + )?; + + assert!(matches!( + error, + CatalogTransitionError::GenerationExhausted { + source: CatalogGenerationError::Exhausted { current: u64::MAX }, + } + )); + Ok(()) +} + +fn admitted<'catalog, 'segment>( + catalog_bytes: &'catalog [u8], + segment_bytes: &'segment [u8], +) -> Result, Box> { + let catalog = ChecksummedCatalog::decode(catalog_bytes)?; + let segment = AdmittedSegment::decode(segment_bytes, maximum_policy())?; + catalog.admit(&[segment]).map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn replace_u64(target: &mut [u8], offset: usize, value: u64) -> Result<(), Box> { + let end = offset.checked_add(8).ok_or("test offset overflow")?; + target + .get_mut(offset..end) + .ok_or("catalog fixture lacks u64 field")? + .copy_from_slice(&value.to_be_bytes()); + Ok(()) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From db38f81a8acfc2d1ed74026da73156e0d1f24a0c Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 00:06:19 -0700 Subject: [PATCH 07/31] Pin immutable catalog snapshots --- src/adapters/admitted_catalog.rs | 8 +- src/adapters/catalog_decoder.rs | 21 +-- src/adapters/catalog_snapshot.rs | 51 +++++++ src/adapters/catalog_snapshot_admission.rs | 25 ++++ src/adapters/catalog_snapshot_error.rs | 54 ++++++++ src/adapters/checksummed_catalog.rs | 55 ++++++-- src/adapters/checksummed_publication_head.rs | 18 ++- src/adapters/mod.rs | 5 + src/lib.rs | 21 +-- tests/catalog_snapshot.rs | 137 +++++++++++++++++++ tests/publication_head/format_oracle.rs | 37 +++++ 11 files changed, 398 insertions(+), 34 deletions(-) create mode 100644 src/adapters/catalog_snapshot.rs create mode 100644 src/adapters/catalog_snapshot_admission.rs create mode 100644 src/adapters/catalog_snapshot_error.rs create mode 100644 tests/catalog_snapshot.rs create mode 100644 tests/publication_head/format_oracle.rs diff --git a/src/adapters/admitted_catalog.rs b/src/adapters/admitted_catalog.rs index 883f1c9..51af0f5 100644 --- a/src/adapters/admitted_catalog.rs +++ b/src/adapters/admitted_catalog.rs @@ -4,9 +4,9 @@ use super::{ AdmittedSegmentRecord, CatalogRecordBinding, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, SegmentRecordIdentity, catalog_transition, }; -use crate::{CatalogDigest, CatalogGeneration}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; -/// Immutable catalog snapshot over exact content-admitted segment records. +/// Immutable admitted catalog over exact content-admitted segment records. /// /// Lookups expose logical identities and verified record bytes. Physical /// segment names, offsets, and lengths remain representation details. @@ -34,6 +34,10 @@ impl<'catalog, 'records> AdmittedCatalog<'catalog, 'records> { self.catalog.previous_catalog_digest() } + pub(crate) const fn length(&self) -> CatalogLength { + self.catalog.length() + } + /// Returns the exact number of logical record bindings. #[must_use] pub const fn record_count(&self) -> u64 { diff --git a/src/adapters/catalog_decoder.rs b/src/adapters/catalog_decoder.rs index 3a6f4c8..894149d 100644 --- a/src/adapters/catalog_decoder.rs +++ b/src/adapters/catalog_decoder.rs @@ -2,7 +2,7 @@ use super::{ CatalogDecodeError, ChecksummedCatalog, catalog_entry_sequence, catalog_header_decoder, - catalog_integrity, + catalog_integrity, checksummed_catalog::CatalogMetadata, }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; @@ -14,22 +14,20 @@ const ALGORITHM: u8 = 1; pub(super) fn decode(encoded: &[u8]) -> Result, CatalogDecodeError> { let fields = catalog_header_decoder::decode(encoded)?; - let (generation, predecessor, catalog_length) = validate_header(&fields)?; - validate_observed_length(encoded, catalog_length)?; - catalog_entry_sequence::validate(encoded, fields.entry_count)?; + let metadata = validate_header(&fields)?; + validate_observed_length(encoded, metadata.length())?; + catalog_entry_sequence::validate(encoded, metadata.entry_count())?; let digest = catalog_integrity::validate(encoded)?; Ok(ChecksummedCatalog::from_verified_parts( encoded, - generation, - predecessor, - fields.entry_count, + metadata, CatalogDigest::from_validated(digest), )) } fn validate_header( fields: &catalog_header_decoder::DecodedCatalogHeader, -) -> Result<(CatalogGeneration, Option, CatalogLength), CatalogDecodeError> { +) -> Result { validate_fixed_fields(fields)?; let generation = CatalogGeneration::new(fields.generation) .map_err(|source| CatalogDecodeError::Generation { source })?; @@ -43,7 +41,12 @@ fn validate_header( let catalog_length = CatalogLength::new(fields.catalog_length) .map_err(|source| CatalogDecodeError::CatalogLength { source })?; validate_count_length(fields.entry_count, catalog_length)?; - Ok((generation, predecessor, catalog_length)) + Ok(CatalogMetadata::new( + generation, + predecessor, + fields.entry_count, + catalog_length, + )) } fn validate_fixed_fields( diff --git a/src/adapters/catalog_snapshot.rs b/src/adapters/catalog_snapshot.rs new file mode 100644 index 0000000..a7dfa0e --- /dev/null +++ b/src/adapters/catalog_snapshot.rs @@ -0,0 +1,51 @@ +//! Immutable reader snapshot pinned by one head and admitted catalog. + +use super::{ + AdmittedCatalog, AdmittedSegmentRecord, ChecksummedPublicationHead, SegmentRecordIdentity, +}; +use crate::{CatalogDigest, CatalogGeneration}; + +/// One complete immutable catalog generation pinned by a checksummed head. +/// +/// The snapshot owns both proofs. Later reads of the mutable head cannot change +/// its generation, logical bindings, or borrowed record bytes. +#[must_use] +#[derive(Debug)] +pub struct CatalogSnapshot<'head, 'catalog, 'records> { + head: ChecksummedPublicationHead<'head>, + catalog: AdmittedCatalog<'catalog, 'records>, +} + +impl<'head, 'catalog, 'records> CatalogSnapshot<'head, 'catalog, 'records> { + /// Returns the exact pinned generation. + pub const fn generation(&self) -> CatalogGeneration { + self.head.generation() + } + + /// Returns the verified physical digest pinned by the head. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.head.catalog_digest() + } + + /// Returns the exact number of logical record bindings. + #[must_use] + pub const fn record_count(&self) -> u64 { + self.catalog.record_count() + } + + /// Looks up one logical record within this pinned generation. + #[must_use] + pub fn record( + &self, + identity: SegmentRecordIdentity, + ) -> Option> { + self.catalog.record(identity) + } + + pub(super) const fn new( + head: ChecksummedPublicationHead<'head>, + catalog: AdmittedCatalog<'catalog, 'records>, + ) -> Self { + Self { head, catalog } + } +} diff --git a/src/adapters/catalog_snapshot_admission.rs b/src/adapters/catalog_snapshot_admission.rs new file mode 100644 index 0000000..4b8b010 --- /dev/null +++ b/src/adapters/catalog_snapshot_admission.rs @@ -0,0 +1,25 @@ +//! Exact publication-head to admitted-catalog snapshot binding. + +use super::{AdmittedCatalog, CatalogSnapshot, CatalogSnapshotError, ChecksummedPublicationHead}; + +pub(super) fn admit<'head, 'catalog, 'records>( + head: ChecksummedPublicationHead<'head>, + catalog: AdmittedCatalog<'catalog, 'records>, +) -> Result, CatalogSnapshotError> { + let expected = head.generation(); + let observed = catalog.generation(); + if observed != expected { + return Err(CatalogSnapshotError::Generation { expected, observed }); + } + let expected = head.catalog_length(); + let observed = catalog.length(); + if observed != expected { + return Err(CatalogSnapshotError::CatalogLength { expected, observed }); + } + let expected = head.catalog_digest(); + let observed = catalog.digest(); + if observed != expected { + return Err(CatalogSnapshotError::CatalogDigest { expected, observed }); + } + Ok(CatalogSnapshot::new(head, catalog)) +} diff --git a/src/adapters/catalog_snapshot_error.rs b/src/adapters/catalog_snapshot_error.rs new file mode 100644 index 0000000..abeb134 --- /dev/null +++ b/src/adapters/catalog_snapshot_error.rs @@ -0,0 +1,54 @@ +//! Publication-head to catalog-snapshot admission failures. + +use std::error::Error; +use std::fmt; + +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Failure to bind one checksummed head to one fully admitted catalog. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogSnapshotError { + /// Head and catalog generations disagreed. + Generation { + /// Generation named by the head. + expected: CatalogGeneration, + /// Generation verified from the catalog. + observed: CatalogGeneration, + }, + /// Head and catalog byte lengths disagreed. + CatalogLength { + /// Catalog length named by the head. + expected: CatalogLength, + /// Verified catalog length. + observed: CatalogLength, + }, + /// Head and catalog physical digests disagreed. + CatalogDigest { + /// Catalog digest named by the head. + expected: CatalogDigest, + /// Verified catalog digest. + observed: CatalogDigest, + }, +} + +impl fmt::Display for CatalogSnapshotError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Generation { expected, observed } => write!( + formatter, + "head generation {} disagrees with catalog generation {}", + expected.get(), + observed.get() + ), + Self::CatalogLength { expected, observed } => write!( + formatter, + "head catalog length {} disagrees with verified length {}", + expected.get(), + observed.get() + ), + Self::CatalogDigest { .. } => formatter.write_str("head and catalog digests disagree"), + } + } +} + +impl Error for CatalogSnapshotError {} diff --git a/src/adapters/checksummed_catalog.rs b/src/adapters/checksummed_catalog.rs index 898fd52..a4e097b 100644 --- a/src/adapters/checksummed_catalog.rs +++ b/src/adapters/checksummed_catalog.rs @@ -4,7 +4,7 @@ use super::{ AdmittedCatalog, AdmittedSegment, CatalogAdmissionError, CatalogDecodeError, CatalogEntries, catalog_admission, catalog_decoder, }; -use crate::{CatalogDigest, CatalogGeneration}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; /// Borrowed catalog bytes with canonical framing, ordering, and integrity proof. /// @@ -15,10 +15,40 @@ use crate::{CatalogDigest, CatalogGeneration}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ChecksummedCatalog<'a> { encoded: &'a [u8], + metadata: CatalogMetadata, + digest: CatalogDigest, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct CatalogMetadata { generation: CatalogGeneration, previous_catalog_digest: Option, entry_count: u64, - digest: CatalogDigest, + length: CatalogLength, +} + +impl CatalogMetadata { + pub(super) const fn new( + generation: CatalogGeneration, + previous_catalog_digest: Option, + entry_count: u64, + length: CatalogLength, + ) -> Self { + Self { + generation, + previous_catalog_digest, + entry_count, + length, + } + } + + pub(super) const fn entry_count(self) -> u64 { + self.entry_count + } + + pub(super) const fn length(self) -> CatalogLength { + self.length + } } impl<'a> ChecksummedCatalog<'a> { @@ -61,19 +91,24 @@ impl<'a> ChecksummedCatalog<'a> { /// Returns the positive catalog generation. pub const fn generation(self) -> CatalogGeneration { - self.generation + self.metadata.generation } /// Returns the predecessor witness, absent only for generation 1. #[must_use] pub const fn previous_catalog_digest(self) -> Option { - self.previous_catalog_digest + self.metadata.previous_catalog_digest } /// Returns the exact bounded entry count. #[must_use] pub const fn entry_count(self) -> u64 { - self.entry_count + self.metadata.entry_count + } + + /// Returns the exact canonical catalog byte length. + pub const fn length(self) -> CatalogLength { + self.metadata.length } /// Returns the verified physical catalog digest. @@ -82,21 +117,17 @@ impl<'a> ChecksummedCatalog<'a> { } pub(super) fn entries(self) -> Result, CatalogDecodeError> { - CatalogEntries::new(self.encoded, self.entry_count) + CatalogEntries::new(self.encoded, self.metadata.entry_count) } pub(super) const fn from_verified_parts( encoded: &'a [u8], - generation: CatalogGeneration, - previous_catalog_digest: Option, - entry_count: u64, + metadata: CatalogMetadata, digest: CatalogDigest, ) -> Self { Self { encoded, - generation, - previous_catalog_digest, - entry_count, + metadata, digest, } } diff --git a/src/adapters/checksummed_publication_head.rs b/src/adapters/checksummed_publication_head.rs index 87f8600..d5ad538 100644 --- a/src/adapters/checksummed_publication_head.rs +++ b/src/adapters/checksummed_publication_head.rs @@ -1,6 +1,9 @@ //! Framing- and checksum-verified borrowed publication head. -use super::{PublicationHeadDecodeError, publication_head_decoder}; +use super::{ + AdmittedCatalog, CatalogSnapshot, CatalogSnapshotError, PublicationHeadDecodeError, + catalog_snapshot_admission, publication_head_decoder, +}; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; /// Borrowed publication-head bytes with canonical framing and checksum proof. @@ -30,6 +33,19 @@ impl<'a> ChecksummedPublicationHead<'a> { publication_head_decoder::decode(encoded) } + /// Pins this head to one fully admitted catalog generation. + /// + /// # Errors + /// + /// Returns [`CatalogSnapshotError`] when the generation, catalog length, or + /// physical catalog digest differs. + pub fn admit<'catalog, 'records>( + self, + catalog: AdmittedCatalog<'catalog, 'records>, + ) -> Result, CatalogSnapshotError> { + catalog_snapshot_admission::admit(self, catalog) + } + /// Returns the exact borrowed canonical bytes. #[must_use] pub const fn encoded(self) -> &'a [u8] { diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 7503f2f..ae2f643 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -28,6 +28,9 @@ mod catalog_entry_sequence; mod catalog_header_decoder; mod catalog_integrity; mod catalog_record_binding; +mod catalog_snapshot; +mod catalog_snapshot_admission; +mod catalog_snapshot_error; mod catalog_successor; mod catalog_transition; mod catalog_transition_error; @@ -121,6 +124,8 @@ pub use catalog_admission_error::CatalogAdmissionError; pub use catalog_allocation_phase::CatalogAllocationPhase; pub use catalog_decode_error::CatalogDecodeError; pub use catalog_entry_decode_error::CatalogEntryDecodeError; +pub use catalog_snapshot::CatalogSnapshot; +pub use catalog_snapshot_error::CatalogSnapshotError; pub use catalog_successor::CatalogSuccessor; pub use catalog_transition_error::CatalogTransitionError; pub use checksummed_catalog::ChecksummedCatalog; diff --git a/src/lib.rs b/src/lib.rs index 9aa6749..b67ed81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,16 +22,17 @@ mod reference; pub use adapters::{ AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalLayoutRecord, CatalogAdmissionError, CatalogAllocationPhase, - CatalogDecodeError, CatalogEntryDecodeError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, - FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, - SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + CatalogDecodeError, CatalogEntryDecodeError, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, + SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, + SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, + SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, + SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, + SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, + SegmentWritePhase, StagedSegment, StorageProfileIdParseError, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_snapshot.rs b/tests/catalog_snapshot.rs new file mode 100644 index 0000000..68cdefa --- /dev/null +++ b/tests/catalog_snapshot.rs @@ -0,0 +1,137 @@ +//! Publication-head to immutable catalog snapshot laws. + +#[path = "publication_head/format_oracle.rs"] +mod format_oracle; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedCatalog, AdmittedSegment, CatalogSnapshotError, ChecksummedCatalog, + ChecksummedPublicationHead, ChunkId, LayoutEntryLimit, SegmentReadPolicy, + SegmentRecordIdentity, SegmentRecordLimit, +}; +use support::{decode_hex, require_error}; + +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const GENERATION_TWO_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_DIGEST_FIELD: usize = 40; + +#[test] +fn snapshot_pins_one_complete_generation_across_later_head_reads() -> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let head_bytes = fixture(HEAD_HEX)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog = admitted_catalog(&catalog_bytes, &segment_bytes)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + + let later_head_bytes = fixture(GENERATION_TWO_HEAD_HEX)?; + let later_head = ChecksummedPublicationHead::decode(&later_head_bytes)?; + let identity = SegmentRecordIdentity::Chunk(ChunkId::hash_bytes(&[0])?); + + assert_eq!(snapshot.generation().get(), 1); + assert_eq!(later_head.generation().get(), 2); + assert_eq!( + snapshot + .record(identity) + .ok_or("snapshot omitted its pinned logical record")? + .payload(), + [0] + ); + Ok(()) +} + +#[test] +fn snapshot_requires_the_head_generation_exactly() -> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let head_bytes = fixture(GENERATION_TWO_HEAD_HEX)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog = admitted_catalog(&catalog_bytes, &segment_bytes)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let error = require_error( + head.admit(catalog), + "generation-mismatched snapshot was admitted", + )?; + + assert!(matches!( + error, + CatalogSnapshotError::Generation { + expected, + observed, + } if expected.get() == 2 && observed.get() == 1 + )); + Ok(()) +} + +#[test] +fn snapshot_requires_the_head_catalog_length_exactly() -> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let head_bytes = fixture(BUNDLE_HEAD_HEX)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog = admitted_catalog(&catalog_bytes, &segment_bytes)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let error = require_error( + head.admit(catalog), + "length-mismatched snapshot was admitted", + )?; + + assert!(matches!( + error, + CatalogSnapshotError::CatalogLength { + expected, + observed, + } if expected.get() == 512 && observed.get() == 352 + )); + Ok(()) +} + +#[test] +fn snapshot_requires_the_head_catalog_digest_exactly() -> Result<(), Box> { + let catalog_bytes = fixture(CATALOG_HEX)?; + let mut head_bytes = fixture(HEAD_HEX)?; + *head_bytes + .get_mut(CATALOG_DIGEST_FIELD) + .ok_or("head fixture lacks catalog digest")? ^= 1; + format_oracle::seal(&mut head_bytes)?; + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog = admitted_catalog(&catalog_bytes, &segment_bytes)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let expected = head.catalog_digest(); + let observed = catalog.digest(); + let error = require_error( + head.admit(catalog), + "digest-mismatched snapshot was admitted", + )?; + + assert!(matches!( + error, + CatalogSnapshotError::CatalogDigest { + expected: error_expected, + observed: error_observed, + } if error_expected == expected && error_observed == observed + )); + Ok(()) +} + +fn admitted_catalog<'catalog, 'segment>( + catalog_bytes: &'catalog [u8], + segment_bytes: &'segment [u8], +) -> Result, Box> { + let catalog = ChecksummedCatalog::decode(catalog_bytes)?; + let segment = AdmittedSegment::decode(segment_bytes, maximum_policy())?; + catalog.admit(&[segment]).map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/publication_head/format_oracle.rs b/tests/publication_head/format_oracle.rs new file mode 100644 index 0000000..0e21e53 --- /dev/null +++ b/tests/publication_head/format_oracle.rs @@ -0,0 +1,37 @@ +//! Independent publication-head checksum test oracle. + +#![allow( + clippy::redundant_pub_crate, + reason = "the snapshot law module consumes this private test oracle" +)] + +use std::error::Error; + +use blake3::Hasher; + +const VERSION: u16 = 1; +const ALGORITHM: u8 = 1; +const ENCODED_LENGTH: usize = 128; +const CHECKSUM_OFFSET: usize = 96; +const CHECKSUM_DOMAIN: &[u8] = b"KEEP:CATHEAD:SUM\0"; + +pub(crate) fn seal(encoded: &mut [u8]) -> Result<(), Box> { + if encoded.len() != ENCODED_LENGTH { + return Err("test publication head has the wrong width".into()); + } + let covered = encoded + .get(..CHECKSUM_OFFSET) + .ok_or("test publication head lacks checksum input")?; + let mut hasher = Hasher::new(); + hasher.update(CHECKSUM_DOMAIN); + hasher.update(&VERSION.to_be_bytes()); + hasher.update(&[ALGORITHM]); + hasher.update(covered); + hasher.update(&u64::try_from(covered.len())?.to_be_bytes()); + let checksum = *hasher.finalize().as_bytes(); + encoded + .get_mut(CHECKSUM_OFFSET..) + .ok_or("test publication head lacks checksum field")? + .copy_from_slice(&checksum); + Ok(()) +} From 9f13b6b68676089b4d072bcb9d2886c20082c582 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 00:18:39 -0700 Subject: [PATCH 08/31] Enforce persistent writer exclusion --- Cargo.lock | 2 + Cargo.toml | 2 + deny.toml | 2 + .../cap-std-and-cap-fs-ext-4.0.2.md | 43 +++++++--- src/adapters/filesystem_writer_lock.rs | 72 ++++++++++++++++ src/adapters/mod.rs | 6 ++ src/adapters/writer_lock_acquire_error.rs | 48 +++++++++++ src/adapters/writer_lock_acquire_phase.rs | 27 ++++++ src/lib.rs | 11 ++- tests/catalog_writer_lock.rs | 86 +++++++++++++++++++ 10 files changed, 282 insertions(+), 17 deletions(-) create mode 100644 src/adapters/filesystem_writer_lock.rs create mode 100644 src/adapters/writer_lock_acquire_error.rs create mode 100644 src/adapters/writer_lock_acquire_phase.rs create mode 100644 tests/catalog_writer_lock.rs diff --git a/Cargo.lock b/Cargo.lock index 5775a7d..91eea88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,6 +325,8 @@ version = "0.0.0" dependencies = [ "allocation-counter", "blake3", + "cap-fs-ext", + "cap-std", "divan", ] diff --git a/Cargo.toml b/Cargo.toml index 7049a72..db24b42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,8 @@ publish = false [dependencies] blake3 = { version = "=1.8.5", default-features = false, features = ["pure", "std"] } +cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"] } +cap-std = { version = "=4.0.2", default-features = false } [dev-dependencies] allocation-counter = { version = "=0.8.1", default-features = false } diff --git a/deny.toml b/deny.toml index b10b587..14021c2 100644 --- a/deny.toml +++ b/deny.toml @@ -14,6 +14,8 @@ allow = [ exceptions = [ # `arrayref` is a required transitive dependency of locked BLAKE3 1.8.5. { allow = ["BSD-2-Clause"], crate = "arrayref@0.3.9" }, + # `winx` is a required transitive dependency of locked `cap-primitives` 4.0.2. + { allow = ["Apache-2.0 WITH LLVM-exception"], crate = "winx@0.36.4" }, ] [sources] diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index b4ab0a5..4c7c9f9 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -1,6 +1,6 @@ # Dependency Admission: cap-std, cap-fs-ext 4.0.2, and rustix 1.1.4 -- Status: Accepted for repository-task filesystem boundaries only +- Status: Accepted for repository-task and segment-store filesystem boundaries - Date: 2026-07-26 - Owner: Keep repository verification - Upstream: @@ -8,9 +8,10 @@ ## Admitted use -Keep admits the exactly pinned `cap-std` 4.0.2, `cap-fs-ext` 4.0.2, and -`rustix` 1.1.4 packages only behind the `xtask` crate's `repository-tasks` -feature. +Keep admits the exactly pinned `cap-std` 4.0.2 and `cap-fs-ext` 4.0.2 packages +for the library's segment-store filesystem adapter and behind the `xtask` +crate's `repository-tasks` feature. Rustix 1.1.4 remains admitted only behind +that `xtask` feature. `cap-std::fs::Dir` pins the admitted repository or corpus directory and opens entries relative to that capability. `cap-fs-ext` supplies no-follow and @@ -21,9 +22,11 @@ these operations let repository checks refuse persistent root replacement, path substitution, symlinked protocol tables, FIFOs, sockets, devices, and other ambiguous filesystem state before reading source or protocol bytes. -These packages are absent from Keep's published library graph, public API, -content identities, durable formats, and production behavior. No -dependency-owned type crosses out of the private repository-task adapter. +The capability packages are present in Keep's published library graph and +production filesystem behavior. No dependency-owned type crosses Keep's public +API or enters content identities or durable formats. The segment-store writer +lock retains capability and file handles behind `FilesystemWriterLock`; its +public acquisition boundary accepts only `std::path::Path`. The bounded subprocess adapter uses Rustix's safe filesystem API to mark child stdin nonblocking before deadline-bounded input transfer. It uses Rustix's safe @@ -64,8 +67,10 @@ work, and require unsafe code that Keep otherwise forbids. All three direct dependencies disable default features. Keep enables only `cap-fs-ext`'s `std` feature and Rustix's `fs`, `process`, and `std` features; -`cap-std` has no enabled feature. All declarations are optional and are -activated solely by `repository-tasks`. +`cap-std` has no enabled feature. The library's capability dependencies are +unconditional because the production segment-store adapter requires them. +The `xtask` declarations remain optional and are activated solely by +`repository-tasks`; Rustix is not a direct library dependency. The locked non-Windows graph introduced for this boundary is: @@ -86,12 +91,18 @@ The locked non-Windows graph introduced for this boundary is: Windows resolution additionally retains the locked `windows-sys`, `windows-targets`, and architecture packages recorded in `Cargo.lock`. +The library therefore exempts only Clippy's `multiple_crate_versions` cargo +lint; exact direct versions, the committed lockfile, dependency policy, and +advisory checks remain authoritative. ## Safety, licensing, and compatibility The capability packages declare `Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT`; Keep selects an admitted license through repository policy. Rustix declares `Apache-2.0 OR MIT`. +The locked `winx` 0.36.4 transitive package declares only +`Apache-2.0 WITH LLVM-exception`, so `deny.toml` admits that exact package and +license combination rather than broadening the global license allowlist. Their manifests declare no Rust-version floor. Compatibility is therefore established only by Keep's pinned stable, MSRV, debug, release, Clippy, dependency-policy, and advisory lanes. @@ -106,11 +117,15 @@ dependencies. ## Failure and recovery boundaries -An open, metadata, read, descriptor-duplication, descriptor-flag, -child-directory setup, child-spawn, stdin-write, output-collection, deadline, -or cleanup failure is a typed refusal. The task never repairs, rewrites, or -substitutes repository data. Retained handles exist only for one verification -process and carry no durability or recovery semantics. +An open, metadata, read, writer-lock acquisition, descriptor-duplication, +descriptor-flag, child-directory setup, child-spawn, stdin-write, +output-collection, deadline, or cleanup failure is a typed refusal. +Repository tasks never repair, rewrite, or substitute repository data. +Repository-task handles exist only for one verification process and carry no +durability or recovery semantics. `FilesystemWriterLock` retains the pinned +store root and persistent lock-file handles for the complete writer-authority +lifetime; dropping the guard releases only the kernel lock and never mutates +the lock file. Keep can remove these dependencies without changing public or durable behavior by replacing them with an equally portable, safe implementation that preserves diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs new file mode 100644 index 0000000..048bbe3 --- /dev/null +++ b/src/adapters/filesystem_writer_lock.rs @@ -0,0 +1,72 @@ +//! Persistent, capability-relative filesystem writer exclusion. + +use std::fs::{File, TryLockError}; +use std::path::Path; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::ambient_authority; +use cap_std::fs::{Dir, OpenOptions}; + +use super::{WriterLockAcquireError, WriterLockAcquirePhase}; + +const LOCK_FILE_NAME: &str = "writer.lock"; + +/// Exclusive kernel-managed writer authority over one pinned store root. +/// +/// The guard retains both the opened root capability and lock-file handle. +/// Dropping it closes the handle and releases the process-scoped kernel lock; +/// it never deletes, renames, truncates, or replaces `writer.lock`. +#[must_use] +pub struct FilesystemWriterLock { + _directory: Dir, + _lock_file: File, +} + +impl FilesystemWriterLock { + /// Tries to acquire exclusive writer authority without blocking. + /// + /// The store root is pinned before `writer.lock` is opened relative to it. + /// The lock entry must already exist as a regular file and is opened + /// without following symbolic links. + /// + /// # Errors + /// + /// Returns [`WriterLockAcquireError::Busy`] when another handle or process + /// owns the lock. Other failures preserve their exact acquisition phase and + /// I/O source. A missing lock file is never created by this operation. + pub fn try_acquire(store_root: &Path) -> Result { + let directory = + Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::OpenRoot, source) + })?; + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .follow(FollowSymlinks::No) + .nonblock(true); + let lock_file = directory + .open_with(LOCK_FILE_NAME, &options) + .map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::OpenFile, source) + })?; + let metadata = lock_file.metadata().map_err(|source| { + WriterLockAcquireError::io(WriterLockAcquirePhase::InspectFile, source) + })?; + if !metadata.is_file() { + return Err(WriterLockAcquireError::NotRegular); + } + let lock_file = lock_file.into_std(); + match lock_file.try_lock() { + Ok(()) => Ok(Self { + _directory: directory, + _lock_file: lock_file, + }), + Err(TryLockError::WouldBlock) => Err(WriterLockAcquireError::Busy), + Err(TryLockError::Error(source)) => Err(WriterLockAcquireError::io( + WriterLockAcquirePhase::Acquire, + source, + )), + } + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index ae2f643..1ee6674 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -39,6 +39,7 @@ mod checksummed_publication_head; mod checksummed_segment_record; mod decoded_catalog_entry; mod filesystem_segment_stage; +mod filesystem_writer_lock; mod framed_blake3; mod layout_decode_error; mod layout_decode_error_display; @@ -114,6 +115,8 @@ mod segment_write_phase; mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; +mod writer_lock_acquire_error; +mod writer_lock_acquire_phase; pub use admitted_catalog::AdmittedCatalog; pub use admitted_segment::AdmittedSegment; @@ -132,6 +135,7 @@ pub use checksummed_catalog::ChecksummedCatalog; pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; pub use filesystem_segment_stage::FilesystemSegmentStage; +pub use filesystem_writer_lock::FilesystemWriterLock; pub use layout_decode_error::LayoutDecodeError; pub use layout_decode_policy::LayoutDecodePolicy; pub use layout_encode_error::LayoutEncodeError; @@ -163,6 +167,8 @@ pub use segment_write_error::SegmentWriteError; pub use segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use staged_segment::StagedSegment; pub use storage_profile_id_text_error::StorageProfileIdParseError; +pub use writer_lock_acquire_error::WriterLockAcquireError; +pub use writer_lock_acquire_phase::WriterLockAcquirePhase; use catalog_entries::CatalogEntries; use catalog_record_binding::CatalogRecordBinding; diff --git a/src/adapters/writer_lock_acquire_error.rs b/src/adapters/writer_lock_acquire_error.rs new file mode 100644 index 0000000..99df18d --- /dev/null +++ b/src/adapters/writer_lock_acquire_error.rs @@ -0,0 +1,48 @@ +//! Persistent writer-lock acquisition failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::WriterLockAcquirePhase; + +/// Failure to acquire exclusive writer authority over one store root. +#[derive(Debug)] +pub enum WriterLockAcquireError { + /// Another handle or process currently owns the kernel lock. + Busy, + /// `writer.lock` was opened but was not a regular file. + NotRegular, + /// One acquisition phase failed at the filesystem boundary. + Io { + /// Exact operation that failed. + phase: WriterLockAcquirePhase, + /// Preserved filesystem source. + source: io::Error, + }, +} + +impl WriterLockAcquireError { + pub(super) const fn io(phase: WriterLockAcquirePhase, source: io::Error) -> Self { + Self::Io { phase, source } + } +} + +impl fmt::Display for WriterLockAcquireError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Busy => formatter.write_str("writer lock is held by another writer"), + Self::NotRegular => formatter.write_str("writer lock entry is not a regular file"), + Self::Io { phase, .. } => write!(formatter, "writer lock {phase} failed"), + } + } +} + +impl Error for WriterLockAcquireError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Busy | Self::NotRegular => None, + } + } +} diff --git a/src/adapters/writer_lock_acquire_phase.rs b/src/adapters/writer_lock_acquire_phase.rs new file mode 100644 index 0000000..63f3ccd --- /dev/null +++ b/src/adapters/writer_lock_acquire_phase.rs @@ -0,0 +1,27 @@ +//! Persistent writer-lock acquisition phases. + +use std::fmt; + +/// Exact filesystem operation attempted while acquiring writer authority. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WriterLockAcquirePhase { + /// Pin the caller-selected store root. + OpenRoot, + /// Open `writer.lock` relative to the pinned root without following links. + OpenFile, + /// Verify that the opened lock handle names a regular file. + InspectFile, + /// Acquire the nonblocking exclusive kernel lock. + Acquire, +} + +impl fmt::Display for WriterLockAcquirePhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::OpenRoot => "root open", + Self::OpenFile => "file open", + Self::InspectFile => "file inspection", + Self::Acquire => "kernel acquisition", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index b67ed81..bf692ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,10 @@ #![deny(warnings)] #![forbid(unsafe_code)] #![warn(clippy::cargo)] +#![allow( + clippy::multiple_crate_versions, + reason = "the audited capability dependencies retain documented platform-only version overlap" +)] //! Correctness-first content-addressed storage. //! @@ -24,15 +28,16 @@ pub use adapters::{ BlobIdTextParseError, CanonicalLayoutRecord, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEntryDecodeError, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, FilesystemSegmentStage, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + ChecksummedSegmentRecord, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, + LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, - SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + SegmentWritePhase, StagedSegment, StorageProfileIdParseError, WriterLockAcquireError, + WriterLockAcquirePhase, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_writer_lock.rs b/tests/catalog_writer_lock.rs new file mode 100644 index 0000000..c354dfd --- /dev/null +++ b/tests/catalog_writer_lock.rs @@ -0,0 +1,86 @@ +//! Persistent one-writer filesystem lock laws. + +#[path = "segment_filesystem_stage/sandbox.rs"] +pub mod sandbox; +mod support; + +use std::error::Error; +use std::fs; + +use keep::{FilesystemWriterLock, WriterLockAcquireError, WriterLockAcquirePhase}; +use sandbox::TestDirectory; +use support::require_error; + +const LOCK_NAME: &str = "writer.lock"; +const RETAINED_EVIDENCE: &[u8] = b"lock contents prove nothing"; + +#[test] +fn one_persistent_lock_excludes_every_second_writer() -> Result<(), Box> { + let sandbox = initialized_lock("writer-exclusion")?; + let first = FilesystemWriterLock::try_acquire(sandbox.path())?; + let error = require_error( + FilesystemWriterLock::try_acquire(sandbox.path()), + "a second writer acquired the persistent lock", + )?; + + assert!(matches!(error, WriterLockAcquireError::Busy)); + drop(first); + + let successor = FilesystemWriterLock::try_acquire(sandbox.path())?; + drop(successor); + assert_eq!(fs::read(sandbox.path().join(LOCK_NAME))?, RETAINED_EVIDENCE); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn missing_lock_evidence_is_never_created_by_acquisition() -> Result<(), Box> { + let sandbox = TestDirectory::create("writer-lock-missing")?; + let error = require_error( + FilesystemWriterLock::try_acquire(sandbox.path()), + "writer acquisition created a missing lock file", + )?; + + assert!(matches!( + error, + WriterLockAcquireError::Io { + phase: WriterLockAcquirePhase::OpenFile, + ref source, + } if source.kind() == std::io::ErrorKind::NotFound + )); + assert!(!sandbox.path().join(LOCK_NAME).exists()); + sandbox.remove()?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn lock_acquisition_never_follows_a_symbolic_link() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let sandbox = TestDirectory::create("writer-lock-symlink")?; + let target = sandbox.path().join("target.lock"); + fs::write(&target, RETAINED_EVIDENCE)?; + symlink(&target, sandbox.path().join(LOCK_NAME))?; + let error = require_error( + FilesystemWriterLock::try_acquire(sandbox.path()), + "writer acquisition followed a symbolic lock path", + )?; + + assert!(matches!( + error, + WriterLockAcquireError::Io { + phase: WriterLockAcquirePhase::OpenFile, + .. + } + )); + assert_eq!(fs::read(target)?, RETAINED_EVIDENCE); + sandbox.remove()?; + Ok(()) +} + +fn initialized_lock(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::write(sandbox.path().join(LOCK_NAME), RETAINED_EVIDENCE)?; + Ok(sandbox) +} From 14c8afd7d52f0cf962b24c497bd5df4c6e621ce5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 00:33:25 -0700 Subject: [PATCH 09/31] Emit canonical catalogs and heads --- src/adapters/canonical_catalog.rs | 59 ++++++ src/adapters/canonical_publication_head.rs | 27 +++ src/adapters/catalog_decoder.rs | 10 +- src/adapters/catalog_encode_error.rs | 123 ++++++++++++ src/adapters/catalog_encoder.rs | 190 ++++++++++++++++++ src/adapters/catalog_encoding_entry.rs | 65 ++++++ src/adapters/catalog_header_encoding.rs | 38 ++++ src/adapters/catalog_integrity.rs | 17 +- src/adapters/mod.rs | 12 ++ src/adapters/publication_head_decoder.rs | 30 ++- src/adapters/publication_head_encoder.rs | 29 +++ .../segment_record_header_encoding.rs | 16 +- .../segment_record_identity_encoding.rs | 17 ++ src/lib.rs | 7 +- tests/catalog_encoding.rs | 130 ++++++++++++ 15 files changed, 724 insertions(+), 46 deletions(-) create mode 100644 src/adapters/canonical_catalog.rs create mode 100644 src/adapters/canonical_publication_head.rs create mode 100644 src/adapters/catalog_encode_error.rs create mode 100644 src/adapters/catalog_encoder.rs create mode 100644 src/adapters/catalog_encoding_entry.rs create mode 100644 src/adapters/catalog_header_encoding.rs create mode 100644 src/adapters/publication_head_encoder.rs create mode 100644 src/adapters/segment_record_identity_encoding.rs create mode 100644 tests/catalog_encoding.rs diff --git a/src/adapters/canonical_catalog.rs b/src/adapters/canonical_catalog.rs new file mode 100644 index 0000000..16a7750 --- /dev/null +++ b/src/adapters/canonical_catalog.rs @@ -0,0 +1,59 @@ +//! Owned canonical catalog bytes derived from admitted segments. + +use super::checksummed_catalog::CatalogMetadata; +use super::{AdmittedSegment, CatalogEncodeError, ChecksummedCatalog, catalog_encoder}; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Owned canonical version-1 catalog bytes. +/// +/// Construction derives physical record coordinates from fully admitted +/// segments, sorts entries by logical identity, and refuses duplicates. The +/// complete catalog is materialized in memory with the version-1 entry-count +/// and byte-length bounds enforced before allocation. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct CanonicalCatalog { + encoded: Vec, + metadata: CatalogMetadata, + digest: CatalogDigest, +} + +impl CanonicalCatalog { + /// Derives one complete canonical catalog from all records in `segments`. + /// + /// # Errors + /// + /// Returns [`CatalogEncodeError`] for an invalid predecessor law, checked + /// count or length refusal, allocation failure, failed immutable segment + /// revalidation, or duplicate logical identity. + pub fn from_segments( + generation: CatalogGeneration, + previous_catalog_digest: Option, + segments: &[AdmittedSegment<'_>], + ) -> Result { + catalog_encoder::encode(generation, previous_catalog_digest, segments) + } + + /// Returns the complete exact canonical bytes. + #[must_use] + pub fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Borrows the generated catalog with its construction-time integrity proof. + pub fn checksummed(&self) -> ChecksummedCatalog<'_> { + ChecksummedCatalog::from_verified_parts(&self.encoded, self.metadata, self.digest) + } + + pub(super) const fn admitted( + encoded: Vec, + metadata: CatalogMetadata, + digest: CatalogDigest, + ) -> Self { + Self { + encoded, + metadata, + digest, + } + } +} diff --git a/src/adapters/canonical_publication_head.rs b/src/adapters/canonical_publication_head.rs new file mode 100644 index 0000000..e143563 --- /dev/null +++ b/src/adapters/canonical_publication_head.rs @@ -0,0 +1,27 @@ +//! Owned canonical publication-head bytes. + +use super::{ChecksummedCatalog, publication_head_encoder}; + +/// Owned canonical version-1 publication head. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CanonicalPublicationHead { + encoded: [u8; 128], +} + +impl CanonicalPublicationHead { + /// Emits the exact head for one checksum- and digest-verified catalog. + pub fn for_catalog(catalog: ChecksummedCatalog<'_>) -> Self { + publication_head_encoder::encode(catalog) + } + + /// Returns the complete exact canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &[u8; 128] { + &self.encoded + } + + pub(super) const fn admitted(encoded: [u8; 128]) -> Self { + Self { encoded } + } +} diff --git a/src/adapters/catalog_decoder.rs b/src/adapters/catalog_decoder.rs index 894149d..8091141 100644 --- a/src/adapters/catalog_decoder.rs +++ b/src/adapters/catalog_decoder.rs @@ -6,11 +6,11 @@ use super::{ }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; -const MAGIC: [u8; 16] = *b"KEEP:CATALOG:V1\0"; -const VERSION: u16 = 1; -const FLAGS: u16 = 0; -const MAXIMUM_ENTRY_COUNT: u64 = 1_048_576; -const ALGORITHM: u8 = 1; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:CATALOG:V1\0"; +pub(super) const VERSION: u16 = 1; +pub(super) const FLAGS: u16 = 0; +pub(super) const MAXIMUM_ENTRY_COUNT: u64 = 1_048_576; +pub(super) const ALGORITHM: u8 = 1; pub(super) fn decode(encoded: &[u8]) -> Result, CatalogDecodeError> { let fields = catalog_header_decoder::decode(encoded)?; diff --git a/src/adapters/catalog_encode_error.rs b/src/adapters/catalog_encode_error.rs new file mode 100644 index 0000000..0513fa3 --- /dev/null +++ b/src/adapters/catalog_encode_error.rs @@ -0,0 +1,123 @@ +//! Canonical catalog emission failures. + +use std::collections::TryReserveError; +use std::error::Error; +use std::fmt; + +use super::{CatalogDecodeError, SegmentDigest, SegmentReadError, SegmentRecordIdentity}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLengthError}; + +/// Failure to derive one canonical catalog from admitted segments. +#[derive(Debug)] +pub enum CatalogEncodeError { + /// Generation 1 was given a predecessor digest. + UnexpectedPredecessor { + /// Supplied predecessor that generation 1 forbids. + observed: CatalogDigest, + }, + /// A later generation omitted its required predecessor digest. + MissingPredecessor { + /// Later generation being encoded. + generation: CatalogGeneration, + }, + /// Summing admitted segment record counts overflowed. + EntryCountArithmetic, + /// The aggregate record count exceeded the format bound. + EntryCountOutOfBounds { + /// Largest version-1 entry count. + maximum: u64, + /// Exact observed aggregate. + observed: u64, + }, + /// The canonical catalog length was not representable. + CatalogLength { + /// Checked length refusal. + source: CatalogLengthError, + }, + /// The canonical catalog length exceeded the host address space. + HostLength { + /// Protocol length that the host could not represent. + observed: u64, + }, + /// Memory reservation for the exact bounded entry set failed. + Allocation { + /// Exact number of entries being retained. + entry_count: u64, + /// Preserved allocation source. + source: TryReserveError, + }, + /// Generated bytes failed independent canonical decoder verification. + Verification { + /// Preserved decoder refusal. + source: CatalogDecodeError, + }, + /// Immutable segment revalidation failed while deriving coordinates. + Segment { + /// Exact segment being traversed. + segment_digest: SegmentDigest, + /// Preserved segment refusal. + source: Box, + }, + /// Two admitted segments supplied the same logical identity. + DuplicateIdentity { + /// Exact duplicated logical identity. + identity: SegmentRecordIdentity, + }, +} + +impl fmt::Display for CatalogEncodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnexpectedPredecessor { .. } => { + formatter.write_str("catalog generation 1 cannot name a predecessor") + } + Self::MissingPredecessor { generation } => write!( + formatter, + "catalog generation {} requires a predecessor", + generation.get() + ), + Self::EntryCountArithmetic => formatter.write_str("catalog entry count overflowed"), + Self::EntryCountOutOfBounds { maximum, observed } => write!( + formatter, + "catalog entry count {observed} exceeds maximum {maximum}" + ), + Self::CatalogLength { .. } => { + formatter.write_str("canonical catalog length is invalid") + } + Self::HostLength { observed } => write!( + formatter, + "catalog length {observed} exceeds the host address space" + ), + Self::Allocation { entry_count, .. } => { + write!( + formatter, + "catalog allocation for {entry_count} entries failed" + ) + } + Self::Verification { .. } => { + formatter.write_str("generated catalog verification failed") + } + Self::Segment { .. } => formatter.write_str("catalog segment revalidation failed"), + Self::DuplicateIdentity { .. } => { + formatter.write_str("catalog input contains a duplicate logical identity") + } + } + } +} + +impl Error for CatalogEncodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::CatalogLength { source } => Some(source), + Self::Allocation { source, .. } => Some(source), + Self::Verification { source } => Some(source), + Self::Segment { source, .. } => Some(source), + Self::UnexpectedPredecessor { .. } + | Self::MissingPredecessor { .. } + | Self::EntryCountArithmetic + | Self::EntryCountOutOfBounds { .. } + | Self::HostLength { .. } + | Self::DuplicateIdentity { .. } => None, + } + } +} diff --git a/src/adapters/catalog_encoder.rs b/src/adapters/catalog_encoder.rs new file mode 100644 index 0000000..4722bb8 --- /dev/null +++ b/src/adapters/catalog_encoder.rs @@ -0,0 +1,190 @@ +//! Bounded canonical catalog emission from admitted segments. + +use super::checksummed_catalog::CatalogMetadata; +use super::{ + AdmittedSegment, CanonicalCatalog, CatalogEncodeError, CatalogEncodingEntry, + ChecksummedCatalog, catalog_decoder, catalog_header_decoder, catalog_header_encoding, + catalog_integrity, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +const TRAILER_LENGTH: u64 = 64; +const DIGEST_LENGTH: u64 = 32; + +pub(super) fn encode( + generation: CatalogGeneration, + predecessor: Option, + segments: &[AdmittedSegment<'_>], +) -> Result { + validate_predecessor(generation, predecessor)?; + let entry_count = entry_count(segments)?; + let catalog_length = catalog_length(entry_count)?; + let mut entries = collect_entries(segments, entry_count)?; + entries.sort_unstable_by_key(CatalogEncodingEntry::identity); + refuse_duplicates(&entries)?; + let encoded = encode_catalog( + generation, + predecessor, + entry_count, + catalog_length, + &entries, + )?; + admit_encoded(encoded) +} + +fn validate_predecessor( + generation: CatalogGeneration, + predecessor: Option, +) -> Result<(), CatalogEncodeError> { + if generation.get() == 1 { + return predecessor.map_or(Ok(()), |observed| { + Err(CatalogEncodeError::UnexpectedPredecessor { observed }) + }); + } + match predecessor { + Some(digest) if digest.as_bytes() != &[0_u8; 32] => Ok(()), + Some(_) | None => Err(CatalogEncodeError::MissingPredecessor { generation }), + } +} + +fn entry_count(segments: &[AdmittedSegment<'_>]) -> Result { + let mut count = 0_u64; + for segment in segments { + count = count + .checked_add(u64::from(segment.record_count())) + .ok_or(CatalogEncodeError::EntryCountArithmetic)?; + } + if count > catalog_decoder::MAXIMUM_ENTRY_COUNT { + return Err(CatalogEncodeError::EntryCountOutOfBounds { + maximum: catalog_decoder::MAXIMUM_ENTRY_COUNT, + observed: count, + }); + } + Ok(count) +} + +fn catalog_length(entry_count: u64) -> Result { + let entry_bytes = entry_count + .checked_mul(u64::from(catalog_header_decoder::ENTRY_LENGTH)) + .ok_or(CatalogEncodeError::EntryCountArithmetic)?; + let length = entry_bytes + .checked_add(u64::from(catalog_header_decoder::HEADER_LENGTH)) + .and_then(|value| value.checked_add(TRAILER_LENGTH)) + .ok_or(CatalogEncodeError::EntryCountArithmetic)?; + CatalogLength::new(length).map_err(|source| CatalogEncodeError::CatalogLength { source }) +} + +fn collect_entries( + segments: &[AdmittedSegment<'_>], + entry_count: u64, +) -> Result, CatalogEncodeError> { + let capacity = + usize::try_from(entry_count).map_err(|_source| CatalogEncodeError::HostLength { + observed: entry_count, + })?; + let mut entries = Vec::new(); + entries + .try_reserve_exact(capacity) + .map_err(|source| CatalogEncodeError::Allocation { + entry_count, + source, + })?; + for segment in segments { + collect_segment_entries(segment, &mut entries)?; + } + Ok(entries) +} + +fn collect_segment_entries( + segment: &AdmittedSegment<'_>, + entries: &mut Vec, +) -> Result<(), CatalogEncodeError> { + let digest = segment.digest(); + let mut cursor = segment.record_cursor(); + while let Some(located) = + cursor + .next_record() + .map_err(|source| CatalogEncodeError::Segment { + segment_digest: digest, + source: Box::new(source), + })? + { + entries.push(CatalogEncodingEntry::from_located(digest, &located)); + } + cursor + .finish() + .map_err(|source| CatalogEncodeError::Segment { + segment_digest: digest, + source: Box::new(source), + }) +} + +fn refuse_duplicates(entries: &[CatalogEncodingEntry]) -> Result<(), CatalogEncodeError> { + for pair in entries.windows(2) { + let [first, second] = pair else { + continue; + }; + if first.identity() == second.identity() { + return Err(CatalogEncodeError::DuplicateIdentity { + identity: first.identity(), + }); + } + } + Ok(()) +} + +fn encode_catalog( + generation: CatalogGeneration, + predecessor: Option, + entry_count: u64, + catalog_length: CatalogLength, + entries: &[CatalogEncodingEntry], +) -> Result, CatalogEncodeError> { + let host_length = usize::try_from(catalog_length.get()).map_err(|_source| { + CatalogEncodeError::HostLength { + observed: catalog_length.get(), + } + })?; + let mut encoded = Vec::new(); + encoded + .try_reserve_exact(host_length) + .map_err(|source| CatalogEncodeError::Allocation { + entry_count, + source, + })?; + encoded.extend_from_slice(&catalog_header_encoding::encode( + generation, + predecessor, + entry_count, + catalog_length, + )); + for entry in entries { + encoded.extend_from_slice(&entry.encode()); + } + let checksum_length = catalog_length + .get() + .checked_sub(TRAILER_LENGTH) + .ok_or(CatalogEncodeError::EntryCountArithmetic)?; + let checksum = catalog_integrity::checksum(&encoded, checksum_length); + encoded.extend_from_slice(&checksum); + let digest_length = catalog_length + .get() + .checked_sub(DIGEST_LENGTH) + .ok_or(CatalogEncodeError::EntryCountArithmetic)?; + let digest = catalog_integrity::digest(&encoded, digest_length); + encoded.extend_from_slice(&digest); + Ok(encoded) +} + +fn admit_encoded(encoded: Vec) -> Result { + let verified = ChecksummedCatalog::decode(&encoded) + .map_err(|source| CatalogEncodeError::Verification { source })?; + let metadata = CatalogMetadata::new( + verified.generation(), + verified.previous_catalog_digest(), + verified.entry_count(), + verified.length(), + ); + let digest = verified.digest(); + Ok(CanonicalCatalog::admitted(encoded, metadata, digest)) +} diff --git a/src/adapters/catalog_encoding_entry.rs b/src/adapters/catalog_encoding_entry.rs new file mode 100644 index 0000000..5168121 --- /dev/null +++ b/src/adapters/catalog_encoding_entry.rs @@ -0,0 +1,65 @@ +//! Canonical catalog-entry state derived from one admitted segment record. + +use super::segment_record_cursor::LocatedRecord; +use super::segment_record_identity_encoding; +use super::segment_record_kind::SegmentRecordKind; +use super::{ + SegmentDigest, SegmentRecordChecksum, SegmentRecordIdentity, SegmentRecordLength, + SegmentRecordPayloadLength, +}; + +pub(super) const ENCODED_LENGTH: usize = 160; + +pub(super) struct CatalogEncodingEntry { + identity: SegmentRecordIdentity, + segment_digest: SegmentDigest, + record_offset: u64, + record_length: SegmentRecordLength, + payload_length: SegmentRecordPayloadLength, + checksum: SegmentRecordChecksum, +} + +impl CatalogEncodingEntry { + pub(super) const fn from_located( + segment_digest: SegmentDigest, + located: &LocatedRecord<'_>, + ) -> Self { + let record = located.record; + let header = record.header(); + Self { + identity: record.identity(), + segment_digest, + record_offset: located.offset, + record_length: header.record_length(), + payload_length: header.payload_length(), + checksum: record.checksum(), + } + } + + pub(super) const fn identity(&self) -> SegmentRecordIdentity { + self.identity + } + + pub(super) const fn encode(&self) -> [u8; ENCODED_LENGTH] { + let mut encoded = [0_u8; ENCODED_LENGTH]; + let kind = SegmentRecordKind::from_identity(self.identity); + let (kind_field, remaining) = encoded.split_at_mut(1); + kind_field.copy_from_slice(&[kind.code()]); + let (_flags, remaining) = remaining.split_at_mut(1); + let (identity_length, remaining) = remaining.split_at_mut(2); + identity_length.copy_from_slice(&kind.identity_length().to_be_bytes()); + let (identity, remaining) = remaining.split_at_mut(60); + identity.copy_from_slice(&segment_record_identity_encoding::encode(self.identity)); + let (segment_digest, remaining) = remaining.split_at_mut(32); + segment_digest.copy_from_slice(self.segment_digest.as_bytes()); + let (record_offset, remaining) = remaining.split_at_mut(8); + record_offset.copy_from_slice(&self.record_offset.to_be_bytes()); + let (record_length, remaining) = remaining.split_at_mut(8); + record_length.copy_from_slice(&self.record_length.get().to_be_bytes()); + let (payload_length, remaining) = remaining.split_at_mut(8); + payload_length.copy_from_slice(&self.payload_length.get().to_be_bytes()); + let (checksum, _reserved) = remaining.split_at_mut(32); + checksum.copy_from_slice(self.checksum.as_bytes()); + encoded + } +} diff --git a/src/adapters/catalog_header_encoding.rs b/src/adapters/catalog_header_encoding.rs new file mode 100644 index 0000000..e3f31c8 --- /dev/null +++ b/src/adapters/catalog_header_encoding.rs @@ -0,0 +1,38 @@ +//! Canonical version-1 catalog-header emission. + +use super::{catalog_decoder, catalog_header_decoder}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +pub(super) const fn encode( + generation: CatalogGeneration, + predecessor: Option, + entry_count: u64, + catalog_length: CatalogLength, +) -> [u8; catalog_header_decoder::HEADER_LENGTH_BYTES] { + let mut encoded = [0_u8; catalog_header_decoder::HEADER_LENGTH_BYTES]; + let (magic, remaining) = encoded.split_at_mut(16); + magic.copy_from_slice(&catalog_decoder::MAGIC); + let (version, remaining) = remaining.split_at_mut(2); + version.copy_from_slice(&catalog_decoder::VERSION.to_be_bytes()); + let (flags, remaining) = remaining.split_at_mut(2); + flags.copy_from_slice(&catalog_decoder::FLAGS.to_be_bytes()); + let (header_length, remaining) = remaining.split_at_mut(2); + header_length.copy_from_slice(&catalog_header_decoder::HEADER_LENGTH.to_be_bytes()); + let (entry_length, remaining) = remaining.split_at_mut(2); + entry_length.copy_from_slice(&catalog_header_decoder::ENTRY_LENGTH.to_be_bytes()); + let (generation_field, remaining) = remaining.split_at_mut(8); + generation_field.copy_from_slice(&generation.get().to_be_bytes()); + let (predecessor_field, remaining) = remaining.split_at_mut(32); + if let Some(digest) = predecessor { + predecessor_field.copy_from_slice(digest.as_bytes()); + } + let (entry_count_field, remaining) = remaining.split_at_mut(8); + entry_count_field.copy_from_slice(&entry_count.to_be_bytes()); + let (catalog_length_field, remaining) = remaining.split_at_mut(8); + catalog_length_field.copy_from_slice(&catalog_length.get().to_be_bytes()); + let (checksum_algorithm, remaining) = remaining.split_at_mut(1); + checksum_algorithm.copy_from_slice(&[catalog_decoder::ALGORITHM]); + let (digest_algorithm, _reserved) = remaining.split_at_mut(1); + digest_algorithm.copy_from_slice(&[catalog_decoder::ALGORITHM]); + encoded +} diff --git a/src/adapters/catalog_integrity.rs b/src/adapters/catalog_integrity.rs index 3712a31..b937210 100644 --- a/src/adapters/catalog_integrity.rs +++ b/src/adapters/catalog_integrity.rs @@ -18,8 +18,7 @@ pub(super) fn validate(encoded: &[u8]) -> Result<[u8; 32], CatalogDecodeError> { .get(..checksum_offset) .ok_or_else(|| minimum_length(encoded))?; let observed_checksum = read_array(encoded, checksum_offset)?; - let expected_checksum = - framed_blake3::hash(CHECKSUM_DOMAIN, &[covered], admitted_length(covered)?); + let expected_checksum = checksum(covered, admitted_length(covered)?); if observed_checksum != expected_checksum { return Err(CatalogDecodeError::ChecksumMismatch { expected: expected_checksum, @@ -30,11 +29,7 @@ pub(super) fn validate(encoded: &[u8]) -> Result<[u8; 32], CatalogDecodeError> { let digest_input = encoded .get(..digest_offset) .ok_or_else(|| minimum_length(encoded))?; - let expected_digest = framed_blake3::hash( - DIGEST_DOMAIN, - &[digest_input], - admitted_length(digest_input)?, - ); + let expected_digest = digest(digest_input, admitted_length(digest_input)?); if observed_digest != expected_digest { return Err(CatalogDecodeError::DigestMismatch { expected: expected_digest, @@ -44,6 +39,14 @@ pub(super) fn validate(encoded: &[u8]) -> Result<[u8; 32], CatalogDecodeError> { Ok(observed_digest) } +pub(super) fn checksum(covered: &[u8], covered_length: u64) -> [u8; 32] { + framed_blake3::hash(CHECKSUM_DOMAIN, &[covered], covered_length) +} + +pub(super) fn digest(covered: &[u8], covered_length: u64) -> [u8; 32] { + framed_blake3::hash(DIGEST_DOMAIN, &[covered], covered_length) +} + fn admitted_length(bytes: &[u8]) -> Result { u64::try_from(bytes.len()).map_err(|_source| CatalogDecodeError::HashLength { observed: bytes.len(), diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 1ee6674..ae764f9 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -12,6 +12,8 @@ mod blob_id_binary; mod blob_id_binary_error; mod blob_id_text; mod blob_id_text_error; +mod canonical_catalog; +mod canonical_publication_head; mod catalog_admission; mod catalog_admission_error; mod catalog_admission_error_display; @@ -19,6 +21,9 @@ mod catalog_allocation_phase; mod catalog_decode_error; mod catalog_decode_error_display; mod catalog_decoder; +mod catalog_encode_error; +mod catalog_encoder; +mod catalog_encoding_entry; mod catalog_entries; mod catalog_entry_decode_error; mod catalog_entry_decode_error_display; @@ -26,6 +31,7 @@ mod catalog_entry_decoder; mod catalog_entry_fields; mod catalog_entry_sequence; mod catalog_header_decoder; +mod catalog_header_encoding; mod catalog_integrity; mod catalog_record_binding; mod catalog_snapshot; @@ -59,6 +65,7 @@ mod lower_hex; mod publication_head_decode_error; mod publication_head_decode_error_display; mod publication_head_decoder; +mod publication_head_encoder; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -90,6 +97,7 @@ mod segment_record_header_error; mod segment_record_header_error_display; mod segment_record_identity; mod segment_record_identity_admission; +mod segment_record_identity_encoding; mod segment_record_kind; mod segment_record_length; mod segment_record_limit; @@ -123,9 +131,12 @@ pub use admitted_segment::AdmittedSegment; pub use admitted_segment_record::AdmittedSegmentRecord; pub use blob_id_binary_error::BlobIdBinaryParseError; pub use blob_id_text_error::BlobIdTextParseError; +pub use canonical_catalog::CanonicalCatalog; +pub use canonical_publication_head::CanonicalPublicationHead; pub use catalog_admission_error::CatalogAdmissionError; pub use catalog_allocation_phase::CatalogAllocationPhase; pub use catalog_decode_error::CatalogDecodeError; +pub use catalog_encode_error::CatalogEncodeError; pub use catalog_entry_decode_error::CatalogEntryDecodeError; pub use catalog_snapshot::CatalogSnapshot; pub use catalog_snapshot_error::CatalogSnapshotError; @@ -170,6 +181,7 @@ pub use storage_profile_id_text_error::StorageProfileIdParseError; pub use writer_lock_acquire_error::WriterLockAcquireError; pub use writer_lock_acquire_phase::WriterLockAcquirePhase; +use catalog_encoding_entry::CatalogEncodingEntry; use catalog_entries::CatalogEntries; use catalog_record_binding::CatalogRecordBinding; use decoded_catalog_entry::DecodedCatalogEntry; diff --git a/src/adapters/publication_head_decoder.rs b/src/adapters/publication_head_decoder.rs index ac1bd7c..4aae5d5 100644 --- a/src/adapters/publication_head_decoder.rs +++ b/src/adapters/publication_head_decoder.rs @@ -3,13 +3,14 @@ use super::{ChecksummedPublicationHead, PublicationHeadDecodeError, framed_blake3}; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; -const ENCODED_LENGTH: usize = 128; -const MAGIC: [u8; 16] = *b"KEEP:CATHEAD:V1\0"; -const VERSION: u16 = 1; -const FLAGS: u16 = 0; -const HEAD_LENGTH: u16 = 128; -const ALGORITHM: u8 = 1; -const CHECKSUM_INPUT_LENGTH: usize = 96; +pub(super) const ENCODED_LENGTH: usize = 128; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:CATHEAD:V1\0"; +pub(super) const VERSION: u16 = 1; +pub(super) const FLAGS: u16 = 0; +pub(super) const HEAD_LENGTH: u16 = 128; +pub(super) const ALGORITHM: u8 = 1; +pub(super) const CHECKSUM_INPUT_LENGTH: usize = 96; +const CHECKSUM_INPUT_LENGTH_U64: u64 = 96; const CHECKSUM_DOMAIN: &[u8] = b"KEEP:CATHEAD:SUM\0"; pub(super) fn decode( @@ -87,21 +88,16 @@ fn validate_checksum(encoded: &[u8], observed: [u8; 32]) -> Result<(), Publicati expected: ENCODED_LENGTH, observed: encoded.len(), })?; - let expected = framed_blake3::hash( - CHECKSUM_DOMAIN, - &[covered], - u64::try_from(CHECKSUM_INPUT_LENGTH).map_err(|_source| { - PublicationHeadDecodeError::WrongLength { - expected: ENCODED_LENGTH, - observed: encoded.len(), - } - })?, - ); + let expected = checksum(covered); require_eq(observed, expected, |observed| { PublicationHeadDecodeError::ChecksumMismatch { expected, observed } }) } +pub(super) fn checksum(covered: &[u8]) -> [u8; 32] { + framed_blake3::hash(CHECKSUM_DOMAIN, &[covered], CHECKSUM_INPUT_LENGTH_U64) +} + fn decode_fields(encoded: &[u8]) -> Result { Ok(DecodedFields { magic: read_array(encoded, 0)?, diff --git a/src/adapters/publication_head_encoder.rs b/src/adapters/publication_head_encoder.rs new file mode 100644 index 0000000..bb07f6d --- /dev/null +++ b/src/adapters/publication_head_encoder.rs @@ -0,0 +1,29 @@ +//! Canonical publication-head emission from one verified catalog. + +use super::{CanonicalPublicationHead, ChecksummedCatalog, publication_head_decoder as format}; + +pub(super) fn encode(catalog: ChecksummedCatalog<'_>) -> CanonicalPublicationHead { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (covered, checksum) = encoded.split_at_mut(format::CHECKSUM_INPUT_LENGTH); + let (magic, remaining) = covered.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, remaining) = remaining.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (flags, remaining) = remaining.split_at_mut(2); + flags.copy_from_slice(&format::FLAGS.to_be_bytes()); + let (head_length, remaining) = remaining.split_at_mut(2); + head_length.copy_from_slice(&format::HEAD_LENGTH.to_be_bytes()); + let (checksum_algorithm, remaining) = remaining.split_at_mut(1); + checksum_algorithm.copy_from_slice(&[format::ALGORITHM]); + let (digest_algorithm, remaining) = remaining.split_at_mut(1); + digest_algorithm.copy_from_slice(&[format::ALGORITHM]); + let (generation, remaining) = remaining.split_at_mut(8); + generation.copy_from_slice(&catalog.generation().get().to_be_bytes()); + let (catalog_length, remaining) = remaining.split_at_mut(8); + catalog_length.copy_from_slice(&catalog.length().get().to_be_bytes()); + let (catalog_digest, remaining) = remaining.split_at_mut(32); + catalog_digest.copy_from_slice(catalog.digest().as_bytes()); + let (_reserved, _complete) = remaining.split_at_mut(24); + checksum.copy_from_slice(&format::checksum(covered)); + CanonicalPublicationHead::admitted(encoded) +} diff --git a/src/adapters/segment_record_header_encoding.rs b/src/adapters/segment_record_header_encoding.rs index ebe71ee..140e678 100644 --- a/src/adapters/segment_record_header_encoding.rs +++ b/src/adapters/segment_record_header_encoding.rs @@ -1,10 +1,10 @@ //! Canonical segment-record-header emitter. -use super::SegmentRecordIdentity; use super::segment_record_header::{ CHECKSUM_ALGORITHM, ENCODED_LENGTH, FLAGS, HEADER_LENGTH, IDENTITY_ALGORITHM, IDENTITY_VERSION, MAGIC, RECORD_VERSION, SegmentRecordHeader, }; +use super::segment_record_identity_encoding; use super::segment_record_kind::SegmentRecordKind; pub(super) const fn encode(header: SegmentRecordHeader) -> [u8; ENCODED_LENGTH] { @@ -35,18 +35,6 @@ pub(super) const fn encode(header: SegmentRecordHeader) -> [u8; ENCODED_LENGTH] identity_algorithm.copy_from_slice(&[IDENTITY_ALGORITHM]); let (_reserved_prefix, remaining) = remaining.split_at_mut(4); let (identity_slot, _reserved_suffix) = remaining.split_at_mut(60); - encode_identity(identity, identity_slot); + identity_slot.copy_from_slice(&segment_record_identity_encoding::encode(identity)); encoded } - -const fn encode_identity(identity: SegmentRecordIdentity, slot: &mut [u8]) { - match identity { - SegmentRecordIdentity::Chunk(id) => { - let (length, remaining) = slot.split_at_mut(4); - length.copy_from_slice(&id.length().get().to_be_bytes()); - let (digest, _unused) = remaining.split_at_mut(32); - digest.copy_from_slice(id.digest()); - } - SegmentRecordIdentity::Layout(id) => slot.copy_from_slice(&id.encode_binary()), - } -} diff --git a/src/adapters/segment_record_identity_encoding.rs b/src/adapters/segment_record_identity_encoding.rs new file mode 100644 index 0000000..8f1fdbb --- /dev/null +++ b/src/adapters/segment_record_identity_encoding.rs @@ -0,0 +1,17 @@ +//! Canonical segment-record identity-slot encoding. + +use super::SegmentRecordIdentity; + +pub(super) const fn encode(identity: SegmentRecordIdentity) -> [u8; 60] { + let mut slot = [0_u8; 60]; + match identity { + SegmentRecordIdentity::Chunk(id) => { + let (length, remaining) = slot.split_at_mut(4); + length.copy_from_slice(&id.length().get().to_be_bytes()); + let (digest, _unused) = remaining.split_at_mut(32); + digest.copy_from_slice(id.digest()); + } + SegmentRecordIdentity::Layout(id) => slot.copy_from_slice(&id.encode_binary()), + } + slot +} diff --git a/src/lib.rs b/src/lib.rs index bf692ba..f04b0f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,9 +25,10 @@ mod reference; pub use adapters::{ AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, - BlobIdTextParseError, CanonicalLayoutRecord, CatalogAdmissionError, CatalogAllocationPhase, - CatalogDecodeError, CatalogEntryDecodeError, CatalogSnapshot, CatalogSnapshotError, - CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, + CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, + CatalogEntryDecodeError, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, + CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, diff --git a/tests/catalog_encoding.rs b/tests/catalog_encoding.rs new file mode 100644 index 0000000..025bbc8 --- /dev/null +++ b/tests/catalog_encoding.rs @@ -0,0 +1,130 @@ +//! Canonical catalog and publication-head emission laws. + +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogEncodeError, + CatalogGeneration, ChecksummedCatalog, ChunkId, LayoutEntryLimit, SegmentReadPolicy, + SegmentRecordIdentity, SegmentRecordLimit, +}; +use support::{decode_hex, require_error}; + +const ONE_ZERO_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const ONE_ZERO_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const GENERATION_TWO_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const ONE_ZERO_HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const GENERATION_TWO_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); + +#[test] +fn admitted_segments_reproduce_every_frozen_catalog() -> Result<(), Box> { + let one_zero_bytes = fixture(ONE_ZERO_SEGMENT_HEX)?; + let bundle_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let one_zero = admitted_segment(&one_zero_bytes)?; + let bundle = admitted_segment(&bundle_bytes)?; + let one_zero_segments = [one_zero]; + let bundle_segments = [bundle]; + + let first = CanonicalCatalog::from_segments(generation(1)?, None, &one_zero_segments)?; + let second = CanonicalCatalog::from_segments( + generation(2)?, + Some(first.checksummed().digest()), + &one_zero_segments, + )?; + let bundled = CanonicalCatalog::from_segments(generation(1)?, None, &bundle_segments)?; + + assert_eq!(first.encoded(), fixture(ONE_ZERO_CATALOG_HEX)?); + assert_eq!(second.encoded(), fixture(GENERATION_TWO_CATALOG_HEX)?); + assert_eq!(bundled.encoded(), fixture(BUNDLE_CATALOG_HEX)?); + Ok(()) +} + +#[test] +fn checksummed_catalogs_reproduce_every_frozen_head() -> Result<(), Box> { + assert_head(ONE_ZERO_CATALOG_HEX, ONE_ZERO_HEAD_HEX)?; + assert_head(GENERATION_TWO_CATALOG_HEX, GENERATION_TWO_HEAD_HEX)?; + assert_head(BUNDLE_CATALOG_HEX, BUNDLE_HEAD_HEX) +} + +#[test] +fn every_generation_enforces_its_exact_predecessor_law() -> Result<(), Box> { + let segment_bytes = fixture(ONE_ZERO_SEGMENT_HEX)?; + let segments = [admitted_segment(&segment_bytes)?]; + let first = CanonicalCatalog::from_segments(generation(1)?, None, &segments)?; + let predecessor = first.checksummed().digest(); + + let missing = require_error( + CanonicalCatalog::from_segments(generation(2)?, None, &segments), + "later catalog omitted its predecessor", + )?; + let unexpected = require_error( + CanonicalCatalog::from_segments(generation(1)?, Some(predecessor), &segments), + "generation 1 admitted a predecessor", + )?; + + assert!(matches!( + missing, + CatalogEncodeError::MissingPredecessor { generation } if generation.get() == 2 + )); + assert!(matches!( + unexpected, + CatalogEncodeError::UnexpectedPredecessor { observed } if observed == predecessor + )); + Ok(()) +} + +#[test] +fn duplicate_logical_records_are_refused_before_emission() -> Result<(), Box> { + let first_bytes = fixture(ONE_ZERO_SEGMENT_HEX)?; + let second_bytes = fixture(ONE_ZERO_SEGMENT_HEX)?; + let segments = [ + admitted_segment(&first_bytes)?, + admitted_segment(&second_bytes)?, + ]; + let expected = SegmentRecordIdentity::Chunk(ChunkId::hash_bytes(&[0])?); + let error = require_error( + CanonicalCatalog::from_segments(generation(1)?, None, &segments), + "duplicate logical records were encoded", + )?; + + assert!(matches!( + error, + CatalogEncodeError::DuplicateIdentity { identity } if identity == expected + )); + Ok(()) +} + +fn assert_head(catalog_hex: &str, head_hex: &str) -> Result<(), Box> { + let catalog_bytes = fixture(catalog_hex)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let head = CanonicalPublicationHead::for_catalog(catalog); + assert_eq!(head.encoded().as_slice(), fixture(head_hex)?); + Ok(()) +} + +fn admitted_segment(bytes: &[u8]) -> Result, Box> { + AdmittedSegment::decode(bytes, maximum_policy()).map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn generation(value: u64) -> Result> { + CatalogGeneration::new(value).map_err(Into::into) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From e8f31c2235ac62e54e449e9201b23a26d3f0a22b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 00:49:57 -0700 Subject: [PATCH 10/31] Order durable catalog publication --- src/adapters/catalog_publication.rs | 118 ++++++++++++ src/adapters/catalog_publication_error.rs | 100 ++++++++++ src/adapters/catalog_publication_execution.rs | 121 ++++++++++++ .../catalog_publication_expectation.rs | 60 ++++++ src/adapters/catalog_publication_phase.rs | 81 ++++++++ src/adapters/catalog_publication_receipt.rs | 33 ++++ src/adapters/catalog_publication_storage.rs | 174 ++++++++++++++++++ src/adapters/mod.rs | 15 ++ src/adapters/segment_publication.rs | 12 ++ src/lib.rs | 25 +-- tests/catalog_publication.rs | 131 +++++++++++++ tests/catalog_publication/preflight_laws.rs | 130 +++++++++++++ .../catalog_publication/recording_storage.rs | 166 +++++++++++++++++ 13 files changed, 1154 insertions(+), 12 deletions(-) create mode 100644 src/adapters/catalog_publication.rs create mode 100644 src/adapters/catalog_publication_error.rs create mode 100644 src/adapters/catalog_publication_execution.rs create mode 100644 src/adapters/catalog_publication_expectation.rs create mode 100644 src/adapters/catalog_publication_phase.rs create mode 100644 src/adapters/catalog_publication_receipt.rs create mode 100644 src/adapters/catalog_publication_storage.rs create mode 100644 src/adapters/segment_publication.rs create mode 100644 tests/catalog_publication.rs create mode 100644 tests/catalog_publication/preflight_laws.rs create mode 100644 tests/catalog_publication/recording_storage.rs diff --git a/src/adapters/catalog_publication.rs b/src/adapters/catalog_publication.rs new file mode 100644 index 0000000..a9276ed --- /dev/null +++ b/src/adapters/catalog_publication.rs @@ -0,0 +1,118 @@ +//! Preflighted catalog-generation publication orchestration. + +use super::catalog_publication_expectation::ExpectedCurrentCatalog; +use super::{ + AdmittedCatalog, AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogTransitionError, ChecksummedPublicationHead, + SegmentPublication, catalog_publication_execution, +}; + +/// Publishes one fully admitted canonical catalog generation. +/// +/// All catalog, segment, and head relationships are verified before the first +/// storage transition. A successful receipt is returned only after atomic head +/// replacement and root-directory synchronization. +/// +/// # Errors +/// +/// Returns [`CatalogPublicationError`] for a staged segment outside the +/// admitted set, catalog or head preflight refusal, snapshot disagreement, or +/// an exact storage transition failure. A failure returns no receipt. +pub fn publish_catalog_generation( + storage: &mut impl CatalogPublicationStorage, + expectation: CatalogPublicationExpectation, + segment: SegmentPublication<'_, '_>, + catalog: &CanonicalCatalog, + segments: &[AdmittedSegment<'_>], +) -> Result { + validate_staged_segment(segment, segments)?; + let checksummed = catalog.checksummed(); + let admitted = checksummed.admit(segments).map_err(|source| { + CatalogPublicationError::CatalogAdmission { + source: Box::new(source), + } + })?; + validate_transition(expectation, &admitted)?; + let head = CanonicalPublicationHead::for_catalog(checksummed); + let checked_head = ChecksummedPublicationHead::decode(head.encoded()) + .map_err(|source| CatalogPublicationError::HeadVerification { source })?; + let snapshot = checked_head + .admit(admitted) + .map_err(|source| CatalogPublicationError::SnapshotAdmission { source })?; + catalog_publication_execution::execute_current(storage, expectation)?; + if let SegmentPublication::One(segment) = segment { + catalog_publication_execution::execute_segment(storage, segment)?; + } + catalog_publication_execution::execute_catalog(storage, catalog, checksummed)?; + catalog_publication_execution::execute_head(storage, &head, &snapshot)?; + Ok(CatalogPublicationReceipt::synchronized( + snapshot.generation(), + snapshot.catalog_digest(), + )) +} + +fn validate_transition( + expectation: CatalogPublicationExpectation, + candidate: &AdmittedCatalog<'_, '_>, +) -> Result<(), CatalogPublicationError> { + match expectation.current() { + ExpectedCurrentCatalog::Uninitialized => validate_initial(candidate), + ExpectedCurrentCatalog::Published { generation, digest } => { + let expected = + generation + .successor() + .map_err(|source| CatalogPublicationError::Transition { + source: CatalogTransitionError::GenerationExhausted { source }, + })?; + if candidate.generation() != expected { + return Err(CatalogPublicationError::Transition { + source: CatalogTransitionError::Generation { + expected, + observed: candidate.generation(), + }, + }); + } + if candidate.previous_catalog_digest() != Some(digest) { + return Err(CatalogPublicationError::Transition { + source: CatalogTransitionError::Predecessor { + expected: digest, + observed: candidate.previous_catalog_digest(), + }, + }); + } + Ok(()) + } + } +} + +const fn validate_initial( + candidate: &AdmittedCatalog<'_, '_>, +) -> Result<(), CatalogPublicationError> { + if candidate.generation().get() != 1 { + return Err(CatalogPublicationError::InitialGeneration { + observed: candidate.generation(), + }); + } + Ok(()) +} + +fn validate_staged_segment( + selection: SegmentPublication<'_, '_>, + segments: &[AdmittedSegment<'_>], +) -> Result<(), CatalogPublicationError> { + let SegmentPublication::One(staged) = selection else { + return Ok(()); + }; + let digest = staged.digest(); + if segments + .iter() + .any(|candidate| candidate.digest() == digest) + { + Ok(()) + } else { + Err(CatalogPublicationError::StagedSegmentNotAdmitted { + segment_digest: digest, + }) + } +} diff --git a/src/adapters/catalog_publication_error.rs b/src/adapters/catalog_publication_error.rs new file mode 100644 index 0000000..0cf928b --- /dev/null +++ b/src/adapters/catalog_publication_error.rs @@ -0,0 +1,100 @@ +//! Catalog-generation publication failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + CatalogAdmissionError, CatalogPublicationPhase, CatalogSnapshotError, + PublicationHeadDecodeError, SegmentDigest, +}; +use crate::{CatalogGeneration, CatalogTransitionError}; + +/// Failure before one catalog generation becomes durably visible. +#[derive(Debug)] +pub enum CatalogPublicationError { + /// An uninitialized store was paired with a later catalog generation. + InitialGeneration { + /// Candidate generation that cannot initialize a store. + observed: CatalogGeneration, + }, + /// A published store was paired with a non-successor candidate. + Transition { + /// Exact successor-law refusal. + source: CatalogTransitionError, + }, + /// The selected segment stage was absent from the admitted segment set. + StagedSegmentNotAdmitted { + /// Exact physical segment digest selected for publication. + segment_digest: SegmentDigest, + }, + /// Catalog locations failed complete segment-record admission. + CatalogAdmission { + /// Preserved admission refusal. + source: Box, + }, + /// Generated next-head bytes failed canonical decoder verification. + HeadVerification { + /// Preserved head decoder refusal. + source: PublicationHeadDecodeError, + }, + /// The verified head and admitted catalog did not bind exactly. + SnapshotAdmission { + /// Preserved snapshot refusal. + source: CatalogSnapshotError, + }, + /// One exact storage transition failed. + Storage { + /// Transition attempted. + phase: CatalogPublicationPhase, + /// Preserved filesystem source. + source: io::Error, + }, +} + +impl CatalogPublicationError { + pub(super) const fn storage(phase: CatalogPublicationPhase, source: io::Error) -> Self { + Self::Storage { phase, source } + } +} + +impl fmt::Display for CatalogPublicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGeneration { observed } => write!( + formatter, + "initial catalog generation must be 1, observed {}", + observed.get() + ), + Self::Transition { .. } => { + formatter.write_str("catalog publication candidate is not the exact successor") + } + Self::StagedSegmentNotAdmitted { .. } => { + formatter.write_str("staged segment is absent from the admitted segment set") + } + Self::CatalogAdmission { .. } => { + formatter.write_str("catalog publication admission failed") + } + Self::HeadVerification { .. } => { + formatter.write_str("generated publication head verification failed") + } + Self::SnapshotAdmission { .. } => { + formatter.write_str("publication snapshot admission failed") + } + Self::Storage { phase, .. } => write!(formatter, "publication {phase} failed"), + } + } +} + +impl Error for CatalogPublicationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Transition { source } => Some(source), + Self::CatalogAdmission { source } => Some(source), + Self::HeadVerification { source } => Some(source), + Self::SnapshotAdmission { source } => Some(source), + Self::Storage { source, .. } => Some(source), + Self::InitialGeneration { .. } | Self::StagedSegmentNotAdmitted { .. } => None, + } + } +} diff --git a/src/adapters/catalog_publication_execution.rs b/src/adapters/catalog_publication_execution.rs new file mode 100644 index 0000000..28a596f --- /dev/null +++ b/src/adapters/catalog_publication_execution.rs @@ -0,0 +1,121 @@ +//! Ordered execution of a fully preflighted catalog publication. + +use std::io; + +use super::{ + CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationPhase, CatalogPublicationStorage, + CatalogSnapshot, ChecksummedCatalog, +}; + +pub(super) fn execute_current( + storage: &mut impl CatalogPublicationStorage, + expectation: CatalogPublicationExpectation, +) -> Result<(), CatalogPublicationError> { + phase( + CatalogPublicationPhase::VerifyCurrent, + storage.verify_current(expectation), + ) +} + +pub(super) fn execute_segment( + storage: &mut impl CatalogPublicationStorage, + segment: &super::AdmittedSegment<'_>, +) -> Result<(), CatalogPublicationError> { + phase( + CatalogPublicationPhase::LinkSegment, + storage.link_segment(segment), + )?; + phase( + CatalogPublicationPhase::VerifySegmentPool, + storage.verify_segment_pool(segment), + )?; + phase( + CatalogPublicationPhase::SynchronizeSegments, + storage.synchronize_segments(), + )?; + phase( + CatalogPublicationPhase::RemoveSegmentStage, + storage.remove_segment_stage(), + )?; + phase( + CatalogPublicationPhase::SynchronizeStagingAfterSegment, + storage.synchronize_staging_after_segment(), + ) +} + +pub(super) fn execute_catalog( + storage: &mut impl CatalogPublicationStorage, + catalog: &CanonicalCatalog, + checksummed: ChecksummedCatalog<'_>, +) -> Result<(), CatalogPublicationError> { + phase( + CatalogPublicationPhase::CreateCatalogStage, + storage.create_catalog_stage(), + )?; + phase( + CatalogPublicationPhase::WriteCatalog, + storage.write_catalog(catalog), + )?; + phase( + CatalogPublicationPhase::FlushCatalog, + storage.flush_catalog(), + )?; + phase( + CatalogPublicationPhase::SynchronizeCatalog, + storage.synchronize_catalog(), + )?; + phase( + CatalogPublicationPhase::LinkCatalog, + storage.link_catalog(checksummed), + )?; + phase( + CatalogPublicationPhase::VerifyCatalogPool, + storage.verify_catalog_pool(checksummed), + )?; + phase( + CatalogPublicationPhase::SynchronizeCatalogs, + storage.synchronize_catalogs(), + )?; + phase( + CatalogPublicationPhase::RemoveCatalogStage, + storage.remove_catalog_stage(), + )?; + phase( + CatalogPublicationPhase::SynchronizeStagingAfterCatalog, + storage.synchronize_staging_after_catalog(), + ) +} + +pub(super) fn execute_head( + storage: &mut impl CatalogPublicationStorage, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, +) -> Result<(), CatalogPublicationError> { + phase( + CatalogPublicationPhase::CreateHeadStage, + storage.create_head_stage(), + )?; + phase(CatalogPublicationPhase::WriteHead, storage.write_head(head))?; + phase(CatalogPublicationPhase::FlushHead, storage.flush_head())?; + phase( + CatalogPublicationPhase::SynchronizeHead, + storage.synchronize_head(), + )?; + phase( + CatalogPublicationPhase::VerifyHeadView, + storage.verify_head_view(head, snapshot), + )?; + phase(CatalogPublicationPhase::ReplaceHead, storage.replace_head())?; + phase( + CatalogPublicationPhase::SynchronizeRoot, + storage.synchronize_root(), + ) +} + +fn phase( + phase: CatalogPublicationPhase, + result: io::Result<()>, +) -> Result<(), CatalogPublicationError> { + result.map_err(|source| CatalogPublicationError::storage(phase, source)) +} diff --git a/src/adapters/catalog_publication_expectation.rs b/src/adapters/catalog_publication_expectation.rs new file mode 100644 index 0000000..91350ec --- /dev/null +++ b/src/adapters/catalog_publication_expectation.rs @@ -0,0 +1,60 @@ +//! Typed current-state expectation for catalog publication. + +use super::CatalogSnapshot; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Current durable state that a writer must revalidate before publication. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatalogPublicationExpectation { + current: ExpectedCurrentCatalog, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ExpectedCurrentCatalog { + Uninitialized, + Published { + generation: CatalogGeneration, + digest: CatalogDigest, + }, +} + +impl CatalogPublicationExpectation { + /// Expects a store with no admitted publication head. + pub const fn uninitialized() -> Self { + Self { + current: ExpectedCurrentCatalog::Uninitialized, + } + } + + /// Expects the exact current coordinates pinned by `snapshot`. + pub const fn successor_of(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Self { + Self { + current: ExpectedCurrentCatalog::Published { + generation: snapshot.generation(), + digest: snapshot.catalog_digest(), + }, + } + } + + /// Returns the expected current generation, absent for an uninitialized store. + #[must_use] + pub const fn current_generation(self) -> Option { + match self.current { + ExpectedCurrentCatalog::Uninitialized => None, + ExpectedCurrentCatalog::Published { generation, .. } => Some(generation), + } + } + + /// Returns the expected current digest, absent for an uninitialized store. + #[must_use] + pub const fn current_catalog_digest(self) -> Option { + match self.current { + ExpectedCurrentCatalog::Uninitialized => None, + ExpectedCurrentCatalog::Published { digest, .. } => Some(digest), + } + } + + pub(super) const fn current(self) -> ExpectedCurrentCatalog { + self.current + } +} diff --git a/src/adapters/catalog_publication_phase.rs b/src/adapters/catalog_publication_phase.rs new file mode 100644 index 0000000..7f57eef --- /dev/null +++ b/src/adapters/catalog_publication_phase.rs @@ -0,0 +1,81 @@ +//! Exact catalog-generation publication durability phases. + +use std::fmt; + +/// Filesystem transition attempted by catalog-generation publication. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogPublicationPhase { + /// Reopen and verify the expected current head and catalog. + VerifyCurrent, + /// Link the synchronized staged segment into the immutable pool. + LinkSegment, + /// Reopen and verify the resolved immutable segment. + VerifySegmentPool, + /// Synchronize the segment-pool directory. + SynchronizeSegments, + /// Remove the fixed segment staging name. + RemoveSegmentStage, + /// Synchronize staging after segment removal. + SynchronizeStagingAfterSegment, + /// Exclusively create the fixed catalog staging name. + CreateCatalogStage, + /// Write the complete canonical catalog. + WriteCatalog, + /// Flush the complete catalog. + FlushCatalog, + /// Synchronize the catalog staging file. + SynchronizeCatalog, + /// Link the verified catalog into the immutable pool. + LinkCatalog, + /// Reopen and verify the resolved immutable catalog. + VerifyCatalogPool, + /// Synchronize the catalog-pool directory. + SynchronizeCatalogs, + /// Remove the fixed catalog staging name. + RemoveCatalogStage, + /// Synchronize staging after catalog removal. + SynchronizeStagingAfterCatalog, + /// Exclusively create `head.next`. + CreateHeadStage, + /// Write the complete canonical next head. + WriteHead, + /// Flush the complete next head. + FlushHead, + /// Synchronize `head.next`. + SynchronizeHead, + /// Reopen and verify the complete transitive head view. + VerifyHeadView, + /// Atomically replace `HEAD` with `head.next`. + ReplaceHead, + /// Synchronize the store root. + SynchronizeRoot, +} + +impl fmt::Display for CatalogPublicationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::VerifyCurrent => "current-head verification", + Self::LinkSegment => "segment link", + Self::VerifySegmentPool => "segment-pool verification", + Self::SynchronizeSegments => "segment-directory synchronization", + Self::RemoveSegmentStage => "segment-stage removal", + Self::SynchronizeStagingAfterSegment => "post-segment staging synchronization", + Self::CreateCatalogStage => "catalog-stage creation", + Self::WriteCatalog => "catalog write", + Self::FlushCatalog => "catalog flush", + Self::SynchronizeCatalog => "catalog synchronization", + Self::LinkCatalog => "catalog link", + Self::VerifyCatalogPool => "catalog-pool verification", + Self::SynchronizeCatalogs => "catalog-directory synchronization", + Self::RemoveCatalogStage => "catalog-stage removal", + Self::SynchronizeStagingAfterCatalog => "post-catalog staging synchronization", + Self::CreateHeadStage => "next-head creation", + Self::WriteHead => "next-head write", + Self::FlushHead => "next-head flush", + Self::SynchronizeHead => "next-head synchronization", + Self::VerifyHeadView => "next-head view verification", + Self::ReplaceHead => "head replacement", + Self::SynchronizeRoot => "root synchronization", + }) + } +} diff --git a/src/adapters/catalog_publication_receipt.rs b/src/adapters/catalog_publication_receipt.rs new file mode 100644 index 0000000..5656ed3 --- /dev/null +++ b/src/adapters/catalog_publication_receipt.rs @@ -0,0 +1,33 @@ +//! Consequential receipt for one fully synchronized catalog generation. + +use crate::{CatalogDigest, CatalogGeneration}; + +/// Proof that publication reached root-directory synchronization. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatalogPublicationReceipt { + generation: CatalogGeneration, + catalog_digest: CatalogDigest, +} + +impl CatalogPublicationReceipt { + /// Returns the exact published catalog generation. + pub const fn generation(self) -> CatalogGeneration { + self.generation + } + + /// Returns the exact published physical catalog digest. + pub const fn catalog_digest(self) -> CatalogDigest { + self.catalog_digest + } + + pub(super) const fn synchronized( + generation: CatalogGeneration, + catalog_digest: CatalogDigest, + ) -> Self { + Self { + generation, + catalog_digest, + } + } +} diff --git a/src/adapters/catalog_publication_storage.rs b/src/adapters/catalog_publication_storage.rs new file mode 100644 index 0000000..ce04803 --- /dev/null +++ b/src/adapters/catalog_publication_storage.rs @@ -0,0 +1,174 @@ +//! Blocking durability port for one writer-locked catalog publication. + +use std::io; + +use super::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogSnapshot, ChecksummedCatalog, +}; + +/// Blocking filesystem capabilities for the catalog publication protocol. +/// +/// An implementation must retain exclusive writer authority and one pinned +/// store root for the complete call. Methods correspond to exact protocol +/// transitions and must not combine later transitions or report success before +/// the named durability or verification obligation is satisfied. +pub trait CatalogPublicationStorage { + /// Reopens and verifies the exact expected current publication state. + /// + /// # Errors + /// + /// Returns the exact current-state verification failure. + fn verify_current(&mut self, expected: CatalogPublicationExpectation) -> io::Result<()>; + + /// Links the exact sealed stage without replacing an immutable pool entry. + /// + /// # Errors + /// + /// Returns the exact link failure. + fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()>; + + /// Reopens and completely verifies the resolved immutable segment. + /// + /// # Errors + /// + /// Returns the exact reopen or verification failure. + fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()>; + + /// Synchronizes the segment-pool directory. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_segments(&mut self) -> io::Result<()>; + + /// Removes only the fixed segment staging name. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_segment_stage(&mut self) -> io::Result<()>; + + /// Synchronizes staging after segment removal. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_staging_after_segment(&mut self) -> io::Result<()>; + + /// Exclusively creates the fixed empty catalog stage. + /// + /// # Errors + /// + /// Returns the exact exclusive-creation failure. + fn create_catalog_stage(&mut self) -> io::Result<()>; + + /// Writes the complete canonical catalog. + /// + /// # Errors + /// + /// Returns the exact write failure. + fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()>; + + /// Flushes the complete catalog stage. + /// + /// # Errors + /// + /// Returns the exact flush failure. + fn flush_catalog(&mut self) -> io::Result<()>; + + /// Synchronizes the complete catalog stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_catalog(&mut self) -> io::Result<()>; + + /// Links the catalog stage without replacing an immutable pool entry. + /// + /// # Errors + /// + /// Returns the exact link failure. + fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()>; + + /// Reopens and completely verifies the resolved immutable catalog. + /// + /// # Errors + /// + /// Returns the exact reopen or verification failure. + fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()>; + + /// Synchronizes the catalog-pool directory. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_catalogs(&mut self) -> io::Result<()>; + + /// Removes only the fixed catalog staging name. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_catalog_stage(&mut self) -> io::Result<()>; + + /// Synchronizes staging after catalog removal. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_staging_after_catalog(&mut self) -> io::Result<()>; + + /// Exclusively creates the fixed empty `head.next`. + /// + /// # Errors + /// + /// Returns the exact exclusive-creation failure. + fn create_head_stage(&mut self) -> io::Result<()>; + + /// Writes the complete canonical next head. + /// + /// # Errors + /// + /// Returns the exact write failure. + fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()>; + + /// Flushes the complete next head. + /// + /// # Errors + /// + /// Returns the exact flush failure. + fn flush_head(&mut self) -> io::Result<()>; + + /// Synchronizes the complete next head. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_head(&mut self) -> io::Result<()>; + + /// Reopens and verifies the exact complete transitive next-head view. + /// + /// # Errors + /// + /// Returns the exact reopen or verification failure. + fn verify_head_view( + &mut self, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result<()>; + + /// Atomically replaces `HEAD` with the verified `head.next`. + /// + /// # Errors + /// + /// Returns the exact atomic-replacement failure. + fn replace_head(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after head replacement. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root(&mut self) -> io::Result<()>; +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index ae764f9..2034e05 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -33,6 +33,13 @@ mod catalog_entry_sequence; mod catalog_header_decoder; mod catalog_header_encoding; mod catalog_integrity; +mod catalog_publication; +mod catalog_publication_error; +mod catalog_publication_execution; +mod catalog_publication_expectation; +mod catalog_publication_phase; +mod catalog_publication_receipt; +mod catalog_publication_storage; mod catalog_record_binding; mod catalog_snapshot; mod catalog_snapshot_admission; @@ -76,6 +83,7 @@ mod segment_header_encoding; mod segment_header_error; mod segment_header_error_display; mod segment_identity_index; +mod segment_publication; mod segment_read_error; mod segment_read_error_display; mod segment_read_policy; @@ -138,6 +146,12 @@ pub use catalog_allocation_phase::CatalogAllocationPhase; pub use catalog_decode_error::CatalogDecodeError; pub use catalog_encode_error::CatalogEncodeError; pub use catalog_entry_decode_error::CatalogEntryDecodeError; +pub use catalog_publication::publish_catalog_generation; +pub use catalog_publication_error::CatalogPublicationError; +pub use catalog_publication_expectation::CatalogPublicationExpectation; +pub use catalog_publication_phase::CatalogPublicationPhase; +pub use catalog_publication_receipt::CatalogPublicationReceipt; +pub use catalog_publication_storage::CatalogPublicationStorage; pub use catalog_snapshot::CatalogSnapshot; pub use catalog_snapshot_error::CatalogSnapshotError; pub use catalog_successor::CatalogSuccessor; @@ -158,6 +172,7 @@ pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; pub use segment_header_error::SegmentHeaderError; +pub use segment_publication::SegmentPublication; pub use segment_read_error::SegmentReadError; pub use segment_read_policy::SegmentReadPolicy; pub use segment_record_admission_error::SegmentRecordAdmissionError; diff --git a/src/adapters/segment_publication.rs b/src/adapters/segment_publication.rs new file mode 100644 index 0000000..69afb47 --- /dev/null +++ b/src/adapters/segment_publication.rs @@ -0,0 +1,12 @@ +//! Optional sealed segment transition preceding catalog publication. + +use super::AdmittedSegment; + +/// Segment-pool work required before publishing one catalog generation. +#[derive(Clone, Copy)] +pub enum SegmentPublication<'selection, 'records> { + /// Every catalog-referenced segment is already durable in the pool. + None, + /// One fixed sealed segment stage must become a durable pool entry. + One(&'selection AdmittedSegment<'records>), +} diff --git a/src/lib.rs b/src/lib.rs index f04b0f6..12ecc0f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,18 +27,19 @@ pub use adapters::{ AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, - CatalogEntryDecodeError, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, - CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, - SegmentHeader, SegmentHeaderError, SegmentReadError, SegmentReadPolicy, - SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, - SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, - SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, - SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, - SegmentWritePhase, StagedSegment, StorageProfileIdParseError, WriterLockAcquireError, - WriterLockAcquirePhase, + CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationPhase, CatalogPublicationReceipt, CatalogPublicationStorage, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + WriterLockAcquireError, WriterLockAcquirePhase, publish_catalog_generation, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_publication.rs b/tests/catalog_publication.rs new file mode 100644 index 0000000..fb7e155 --- /dev/null +++ b/tests/catalog_publication.rs @@ -0,0 +1,131 @@ +//! Catalog-generation publication ordering and fault laws. + +#[path = "catalog_publication/preflight_laws.rs"] +mod preflight_laws; +#[path = "catalog_publication/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationPhase, LayoutEntryLimit, SegmentPublication, + SegmentReadPolicy, SegmentRecordLimit, publish_catalog_generation, +}; +use recording_storage::{EXPECTED_WITH_SEGMENT, RecordingStorage}; +use support::{decode_hex, require_error}; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/empty-segment.hex"); + +#[test] +fn one_generation_executes_every_durability_transition_in_order() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let fixture = publication_fixture(&bytes)?; + let staged = fixture.segments.first().ok_or("missing staged segment")?; + let mut storage = RecordingStorage::succeeding(); + + let receipt = publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(staged), + &fixture.catalog, + &fixture.segments, + )?; + + assert_eq!(storage.observed(), EXPECTED_WITH_SEGMENT); + assert_eq!(receipt.generation().get(), 1); + assert_eq!( + receipt.catalog_digest(), + fixture.catalog.checksummed().digest() + ); + Ok(()) +} + +#[test] +fn every_publication_fault_stops_at_its_exact_phase() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let fixture = publication_fixture(&bytes)?; + let staged = fixture.segments.first().ok_or("missing staged segment")?; + + for failing_phase in EXPECTED_WITH_SEGMENT { + let mut storage = RecordingStorage::failing_at(*failing_phase); + let error = require_error( + publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(staged), + &fixture.catalog, + &fixture.segments, + ), + "faulting publication returned a receipt", + )?; + + assert!(matches!( + error, + CatalogPublicationError::Storage { phase, .. } if phase == *failing_phase + )); + assert_eq!(storage.observed().last(), Some(failing_phase)); + assert_eq!( + storage.observed().len(), + expected_prefix_length(*failing_phase)? + ); + } + Ok(()) +} + +#[test] +fn catalog_only_publication_skips_segment_transitions() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let fixture = publication_fixture(&bytes)?; + let mut storage = RecordingStorage::succeeding(); + + let _receipt = publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::None, + &fixture.catalog, + &fixture.segments, + )?; + + assert_eq!( + storage.observed().get(1), + Some(&CatalogPublicationPhase::CreateCatalogStage) + ); + assert_eq!( + storage.observed().len(), + EXPECTED_WITH_SEGMENT + .len() + .checked_sub(5) + .ok_or("publication phase count underflowed")? + ); + Ok(()) +} + +struct PublicationFixture<'a> { + segments: [AdmittedSegment<'a>; 1], + catalog: CanonicalCatalog, +} + +fn publication_fixture(bytes: &[u8]) -> Result, Box> { + let segments = [AdmittedSegment::decode(bytes, maximum_policy())?]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + Ok(PublicationFixture { segments, catalog }) +} + +fn expected_prefix_length(phase: CatalogPublicationPhase) -> Result> { + EXPECTED_WITH_SEGMENT + .iter() + .position(|candidate| *candidate == phase) + .and_then(|index| index.checked_add(1)) + .ok_or_else(|| "missing expected publication phase".into()) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/catalog_publication/preflight_laws.rs b/tests/catalog_publication/preflight_laws.rs new file mode 100644 index 0000000..2a72d41 --- /dev/null +++ b/tests/catalog_publication/preflight_laws.rs @@ -0,0 +1,130 @@ +//! Publication preflight and current-state transition laws. + +use std::error::Error; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, + CatalogPublicationError, CatalogPublicationExpectation, CatalogTransitionError, + ChecksummedPublicationHead, SegmentPublication, publish_catalog_generation, +}; + +use super::recording_storage::RecordingStorage; +use super::{EMPTY_SEGMENT_HEX, SEGMENT_HEX, fixture, maximum_policy, publication_fixture}; +use crate::support::require_error; + +#[test] +fn staged_segment_must_belong_to_the_admitted_set_before_io() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let publication = publication_fixture(&bytes)?; + let staged_bytes = fixture(EMPTY_SEGMENT_HEX)?; + let staged = AdmittedSegment::decode(&staged_bytes, maximum_policy())?; + let expected = staged.digest(); + let mut storage = RecordingStorage::succeeding(); + let error = require_error( + publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(&staged), + &publication.catalog, + &publication.segments, + ), + "unadmitted staged segment reached publication", + )?; + + assert!(matches!( + error, + CatalogPublicationError::StagedSegmentNotAdmitted { segment_digest } + if segment_digest == expected + )); + assert!(storage.observed().is_empty()); + Ok(()) +} + +#[test] +fn catalog_location_refusal_precedes_every_storage_call() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let publication = publication_fixture(&bytes)?; + let mut storage = RecordingStorage::succeeding(); + let error = require_error( + publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::None, + &publication.catalog, + &[], + ), + "catalog with a missing segment reached publication", + )?; + + assert!(matches!( + error, + CatalogPublicationError::CatalogAdmission { .. } + )); + assert!(storage.observed().is_empty()); + Ok(()) +} + +#[test] +fn current_snapshot_requires_and_admits_only_its_exact_successor() -> Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let publication = publication_fixture(&bytes)?; + let current_catalog = publication.catalog.checksummed(); + let current_head = CanonicalPublicationHead::for_catalog(current_catalog); + let checked_head = ChecksummedPublicationHead::decode(current_head.encoded())?; + let admitted = current_catalog.admit(&publication.segments)?; + let current = checked_head.admit(admitted)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t); + let mut storage = RecordingStorage::succeeding(); + + let stale = require_error( + publish_catalog_generation( + &mut storage, + expectation, + SegmentPublication::None, + &publication.catalog, + &publication.segments, + ), + "current generation was republished as its own successor", + )?; + assert!(matches!( + stale, + CatalogPublicationError::Transition { + source: CatalogTransitionError::Generation { + expected, + observed, + }, + } if expected.get() == 2 && observed.get() == 1 + )); + assert!(storage.observed().is_empty()); + + let successor = CanonicalCatalog::from_segments( + CatalogGeneration::new(2)?, + Some(current.catalog_digest()), + &publication.segments, + )?; + let wrong_initial = require_error( + publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::None, + &successor, + &publication.segments, + ), + "generation 2 initialized an uninitialized store", + )?; + assert!(matches!( + wrong_initial, + CatalogPublicationError::InitialGeneration { observed } if observed.get() == 2 + )); + assert!(storage.observed().is_empty()); + + let receipt = publish_catalog_generation( + &mut storage, + expectation, + SegmentPublication::None, + &successor, + &publication.segments, + )?; + assert_eq!(receipt.generation().get(), 2); + Ok(()) +} diff --git a/tests/catalog_publication/recording_storage.rs b/tests/catalog_publication/recording_storage.rs new file mode 100644 index 0000000..0ad601d --- /dev/null +++ b/tests/catalog_publication/recording_storage.rs @@ -0,0 +1,166 @@ +//! Deterministic fault-recording catalog publication storage. + +use std::io; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogPublicationPhase, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, +}; + +/// Exact complete publication order when one segment stage is present. +pub const EXPECTED_WITH_SEGMENT: &[CatalogPublicationPhase] = &[ + CatalogPublicationPhase::VerifyCurrent, + CatalogPublicationPhase::LinkSegment, + CatalogPublicationPhase::VerifySegmentPool, + CatalogPublicationPhase::SynchronizeSegments, + CatalogPublicationPhase::RemoveSegmentStage, + CatalogPublicationPhase::SynchronizeStagingAfterSegment, + CatalogPublicationPhase::CreateCatalogStage, + CatalogPublicationPhase::WriteCatalog, + CatalogPublicationPhase::FlushCatalog, + CatalogPublicationPhase::SynchronizeCatalog, + CatalogPublicationPhase::LinkCatalog, + CatalogPublicationPhase::VerifyCatalogPool, + CatalogPublicationPhase::SynchronizeCatalogs, + CatalogPublicationPhase::RemoveCatalogStage, + CatalogPublicationPhase::SynchronizeStagingAfterCatalog, + CatalogPublicationPhase::CreateHeadStage, + CatalogPublicationPhase::WriteHead, + CatalogPublicationPhase::FlushHead, + CatalogPublicationPhase::SynchronizeHead, + CatalogPublicationPhase::VerifyHeadView, + CatalogPublicationPhase::ReplaceHead, + CatalogPublicationPhase::SynchronizeRoot, +]; + +/// Storage port that records calls and optionally fails at one exact phase. +pub struct RecordingStorage { + observed: Vec, + failing_phase: Option, +} + +impl RecordingStorage { + /// Creates a recorder that admits every transition. + pub const fn succeeding() -> Self { + Self { + observed: Vec::new(), + failing_phase: None, + } + } + + /// Creates a recorder that refuses one exact transition. + pub const fn failing_at(phase: CatalogPublicationPhase) -> Self { + Self { + observed: Vec::new(), + failing_phase: Some(phase), + } + } + + /// Returns every transition attempted so far. + pub fn observed(&self) -> &[CatalogPublicationPhase] { + &self.observed + } + + fn record(&mut self, phase: CatalogPublicationPhase) -> io::Result<()> { + self.observed.push(phase); + if self.failing_phase == Some(phase) { + Err(io::Error::other("injected publication failure")) + } else { + Ok(()) + } + } +} + +impl CatalogPublicationStorage for RecordingStorage { + fn verify_current(&mut self, _expected: CatalogPublicationExpectation) -> io::Result<()> { + self.record(CatalogPublicationPhase::VerifyCurrent) + } + + fn link_segment(&mut self, _segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.record(CatalogPublicationPhase::LinkSegment) + } + + fn verify_segment_pool(&mut self, _segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.record(CatalogPublicationPhase::VerifySegmentPool) + } + + fn synchronize_segments(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeSegments) + } + + fn remove_segment_stage(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::RemoveSegmentStage) + } + + fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeStagingAfterSegment) + } + + fn create_catalog_stage(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::CreateCatalogStage) + } + + fn write_catalog(&mut self, _catalog: &CanonicalCatalog) -> io::Result<()> { + self.record(CatalogPublicationPhase::WriteCatalog) + } + + fn flush_catalog(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::FlushCatalog) + } + + fn synchronize_catalog(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeCatalog) + } + + fn link_catalog(&mut self, _catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.record(CatalogPublicationPhase::LinkCatalog) + } + + fn verify_catalog_pool(&mut self, _catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.record(CatalogPublicationPhase::VerifyCatalogPool) + } + + fn synchronize_catalogs(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeCatalogs) + } + + fn remove_catalog_stage(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::RemoveCatalogStage) + } + + fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeStagingAfterCatalog) + } + + fn create_head_stage(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::CreateHeadStage) + } + + fn write_head(&mut self, _head: &CanonicalPublicationHead) -> io::Result<()> { + self.record(CatalogPublicationPhase::WriteHead) + } + + fn flush_head(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::FlushHead) + } + + fn synchronize_head(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeHead) + } + + fn verify_head_view( + &mut self, + _head: &CanonicalPublicationHead, + _snapshot: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result<()> { + self.record(CatalogPublicationPhase::VerifyHeadView) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::ReplaceHead) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + self.record(CatalogPublicationPhase::SynchronizeRoot) + } +} From 95c3d3adf839dc64e4b5bc48806c14cdeb239dda Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:08:22 -0700 Subject: [PATCH 11/31] Load published catalog snapshots --- src/adapters/catalog_restart_artifact.rs | 17 ++ src/adapters/catalog_restart_byte_limit.rs | 40 ++++ src/adapters/catalog_restart_error.rs | 198 ++++++++++++++++++++ src/adapters/catalog_restart_io.rs | 92 +++++++++ src/adapters/catalog_restart_loader.rs | 105 +++++++++++ src/adapters/catalog_restart_phase.rs | 32 ++++ src/adapters/catalog_restart_policy.rs | 31 +++ src/adapters/catalog_restart_segments.rs | 109 +++++++++++ src/adapters/filesystem_catalog_snapshot.rs | 122 ++++++++++++ src/adapters/loaded_segment.rs | 23 +++ src/adapters/mod.rs | 17 ++ src/adapters/physical_pool_name.rs | 29 +++ src/catalog/length.rs | 22 ++- src/lib.rs | 25 +-- tests/catalog_restart.rs | 89 +++++++++ tests/catalog_restart/refusal_laws.rs | 139 ++++++++++++++ 16 files changed, 1071 insertions(+), 19 deletions(-) create mode 100644 src/adapters/catalog_restart_artifact.rs create mode 100644 src/adapters/catalog_restart_byte_limit.rs create mode 100644 src/adapters/catalog_restart_error.rs create mode 100644 src/adapters/catalog_restart_io.rs create mode 100644 src/adapters/catalog_restart_loader.rs create mode 100644 src/adapters/catalog_restart_phase.rs create mode 100644 src/adapters/catalog_restart_policy.rs create mode 100644 src/adapters/catalog_restart_segments.rs create mode 100644 src/adapters/filesystem_catalog_snapshot.rs create mode 100644 src/adapters/loaded_segment.rs create mode 100644 src/adapters/physical_pool_name.rs create mode 100644 tests/catalog_restart.rs create mode 100644 tests/catalog_restart/refusal_laws.rs diff --git a/src/adapters/catalog_restart_artifact.rs b/src/adapters/catalog_restart_artifact.rs new file mode 100644 index 0000000..07d21cf --- /dev/null +++ b/src/adapters/catalog_restart_artifact.rs @@ -0,0 +1,17 @@ +//! Restart-loaded physical artifact classifications. + +use super::SegmentDigest; + +/// Physical artifact whose kind, length, or allocation was refused. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogRestartArtifact { + /// Fixed publication head. + Head, + /// Exact head-selected catalog. + Catalog, + /// Exact catalog-selected segment. + Segment { + /// Selected physical digest. + digest: SegmentDigest, + }, +} diff --git a/src/adapters/catalog_restart_byte_limit.rs b/src/adapters/catalog_restart_byte_limit.rs new file mode 100644 index 0000000..4eeafe3 --- /dev/null +++ b/src/adapters/catalog_restart_byte_limit.rs @@ -0,0 +1,40 @@ +//! Caller-selected aggregate restart segment-byte bound. + +use std::error::Error; +use std::fmt; +use std::num::NonZeroU64; + +/// Positive maximum segment bytes retained by one restart snapshot. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatalogRestartByteLimit(NonZeroU64); + +impl CatalogRestartByteLimit { + /// Admits one positive aggregate byte bound. + /// + /// # Errors + /// + /// Returns [`CatalogRestartByteLimitError`] when `value` is zero. + pub const fn new(value: u64) -> Result { + match NonZeroU64::new(value) { + Some(value) => Ok(Self(value)), + None => Err(CatalogRestartByteLimitError), + } + } + + /// Returns the exact caller-selected bound. + pub const fn get(self) -> u64 { + self.0.get() + } +} + +/// Refusal of a zero aggregate restart byte bound. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatalogRestartByteLimitError; + +impl fmt::Display for CatalogRestartByteLimitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("catalog restart byte limit must be positive") + } +} + +impl Error for CatalogRestartByteLimitError {} diff --git a/src/adapters/catalog_restart_error.rs b/src/adapters/catalog_restart_error.rs new file mode 100644 index 0000000..b7e28e8 --- /dev/null +++ b/src/adapters/catalog_restart_error.rs @@ -0,0 +1,198 @@ +//! Published catalog restart-loading failures. + +use std::collections::TryReserveError; +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + CatalogAdmissionError, CatalogDecodeError, CatalogRestartArtifact, CatalogRestartPhase, + CatalogSnapshotError, PublicationHeadDecodeError, SegmentDigest, SegmentReadError, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Failure to reconstruct one exact published catalog snapshot. +#[derive(Debug)] +pub enum CatalogRestartError { + /// One capability-relative filesystem operation failed. + Io { + /// Exact operation that failed. + phase: CatalogRestartPhase, + /// Preserved filesystem source. + source: io::Error, + }, + /// An opened protocol artifact was not a regular file. + NotRegular { + /// Artifact whose type was wrong. + artifact: CatalogRestartArtifact, + }, + /// An artifact length violated an exact or bounded expectation. + Length { + /// Artifact whose length was refused. + artifact: CatalogRestartArtifact, + /// Smallest accepted length. + minimum: u64, + /// Largest accepted length. + maximum: u64, + /// Exact observed length. + observed: u64, + }, + /// Exact length arithmetic could not be represented. + LengthArithmetic { + /// Artifact whose observation could not be represented. + artifact: CatalogRestartArtifact, + /// Length established before the overflow. + expected: u64, + }, + /// Host memory could not represent or reserve a bounded artifact. + Allocation { + /// Artifact being materialized. + artifact: CatalogRestartArtifact, + /// Exact requested byte count. + byte_count: u64, + /// Preserved allocation source when reservation was attempted. + source: Option, + }, + /// A host segment-index length could not be represented as a protocol count. + SegmentIndexLength, + /// Host memory could not reserve the exact segment index. + SegmentIndexAllocation { + /// Exact number of segment entries requested. + segment_count: u64, + /// Preserved allocation source. + source: TryReserveError, + }, + /// Aggregate retained segment bytes exceeded caller policy. + RetainedSegmentBytes { + /// Caller-selected maximum. + maximum: u64, + /// Exact attempted aggregate. + observed: u64, + }, + /// Aggregate retained-segment byte arithmetic overflowed. + RetainedSegmentByteArithmetic { + /// Bytes retained before the failing addition. + current: u64, + /// Bytes selected by the next segment. + addition: u64, + }, + /// Publication-head bytes were malformed or corrupt. + Head { + /// Preserved head decoder refusal. + source: PublicationHeadDecodeError, + }, + /// Catalog bytes were malformed, noncanonical, or corrupt. + Catalog { + /// Preserved catalog decoder refusal. + source: CatalogDecodeError, + }, + /// The selected catalog disagreed with the head coordinate. + CatalogCoordinate { + /// Generation required by the head. + expected_generation: CatalogGeneration, + /// Generation verified from the catalog. + observed_generation: CatalogGeneration, + /// Length required by the head. + expected_length: CatalogLength, + /// Length verified from the catalog. + observed_length: CatalogLength, + /// Digest required by the head. + expected_digest: CatalogDigest, + /// Digest verified from the catalog. + observed_digest: CatalogDigest, + }, + /// One selected segment was malformed or corrupt. + Segment { + /// Digest required by the catalog. + expected: SegmentDigest, + /// Preserved segment refusal. + source: Box, + }, + /// A valid segment's content digest disagreed with its selected name. + SegmentCoordinate { + /// Digest required by the catalog. + expected: SegmentDigest, + /// Digest verified from bytes. + observed: SegmentDigest, + }, + /// Catalog locations failed exact segment-record admission. + CatalogAdmission { + /// Preserved admission refusal. + source: Box, + }, + /// Head and admitted catalog failed final snapshot binding. + Snapshot { + /// Preserved snapshot refusal. + source: CatalogSnapshotError, + }, +} + +impl CatalogRestartError { + pub(super) const fn io(phase: CatalogRestartPhase, source: io::Error) -> Self { + Self::Io { phase, source } + } +} + +impl fmt::Display for CatalogRestartError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { phase, .. } => write!(formatter, "catalog restart {phase} failed"), + Self::NotRegular { .. } => formatter.write_str("restart artifact is not regular"), + Self::Length { .. } => formatter.write_str("restart artifact length is invalid"), + Self::LengthArithmetic { .. } => { + formatter.write_str("restart artifact length overflowed") + } + Self::Allocation { .. } => formatter.write_str("restart allocation failed"), + Self::SegmentIndexLength => { + formatter.write_str("restart segment index length is not representable") + } + Self::SegmentIndexAllocation { .. } => { + formatter.write_str("restart segment index allocation failed") + } + Self::RetainedSegmentBytes { .. } => { + formatter.write_str("retained segment bytes exceed restart policy") + } + Self::RetainedSegmentByteArithmetic { .. } => { + formatter.write_str("retained segment byte arithmetic overflowed") + } + Self::Head { .. } => formatter.write_str("publication head admission failed"), + Self::Catalog { .. } => formatter.write_str("catalog admission failed"), + Self::CatalogCoordinate { .. } => { + formatter.write_str("catalog disagrees with publication head") + } + Self::Segment { .. } => formatter.write_str("segment admission failed"), + Self::SegmentCoordinate { .. } => { + formatter.write_str("segment disagrees with catalog coordinate") + } + Self::CatalogAdmission { .. } => formatter.write_str("catalog record binding failed"), + Self::Snapshot { .. } => formatter.write_str("restart snapshot binding failed"), + } + } +} + +impl Error for CatalogRestartError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Allocation { + source: Some(source), + .. + } + | Self::SegmentIndexAllocation { source, .. } => Some(source), + Self::Head { source } => Some(source), + Self::Catalog { source } => Some(source), + Self::Segment { source, .. } => Some(source), + Self::CatalogAdmission { source } => Some(source), + Self::Snapshot { source } => Some(source), + Self::NotRegular { .. } + | Self::Length { .. } + | Self::LengthArithmetic { .. } + | Self::Allocation { source: None, .. } + | Self::SegmentIndexLength + | Self::RetainedSegmentBytes { .. } + | Self::RetainedSegmentByteArithmetic { .. } + | Self::CatalogCoordinate { .. } + | Self::SegmentCoordinate { .. } => None, + } + } +} diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs new file mode 100644 index 0000000..0a62c94 --- /dev/null +++ b/src/adapters/catalog_restart_io.rs @@ -0,0 +1,92 @@ +//! This module owns exact capability-relative restart artifact reads. + +use std::io::{self, Read}; +use std::path::Path; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::ambient_authority; +use cap_std::fs::{Dir, File, OpenOptions}; + +use super::{CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase}; + +pub(super) fn open_root(root: &Path) -> Result { + Dir::open_ambient_dir(root, ambient_authority()) + .map_err(|source| CatalogRestartError::io(CatalogRestartPhase::OpenRoot, source)) +} + +pub(super) fn open_regular( + directory: &Dir, + name: &str, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, +) -> Result<(File, u64), CatalogRestartError> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + let file = directory + .open_with(name, &options) + .map_err(|source| CatalogRestartError::io(phase, source))?; + let metadata = file + .metadata() + .map_err(|source| CatalogRestartError::io(phase, source))?; + if !metadata.is_file() { + return Err(CatalogRestartError::NotRegular { artifact }); + } + Ok((file, metadata.len())) +} + +pub(super) fn read_exact( + mut file: File, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, +) -> Result, CatalogRestartError> { + let host_length = + usize::try_from(expected).map_err(|_source| CatalogRestartError::Allocation { + artifact, + byte_count: expected, + source: None, + })?; + let mut encoded = Vec::new(); + encoded + .try_reserve_exact(host_length) + .map_err(|source| CatalogRestartError::Allocation { + artifact, + byte_count: expected, + source: Some(source), + })?; + encoded.resize(host_length, 0); + file.read_exact(&mut encoded) + .map_err(|source| CatalogRestartError::io(phase, source))?; + reject_trailing_bytes(&mut file, artifact, phase, expected)?; + Ok(encoded) +} + +fn reject_trailing_bytes( + file: &mut File, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, +) -> Result<(), CatalogRestartError> { + let mut trailing = [0_u8; 1]; + loop { + match file.read(&mut trailing) { + Ok(0) => return Ok(()), + Ok(observed) => { + let increment = u64::try_from(observed).map_err(|_source| { + CatalogRestartError::LengthArithmetic { artifact, expected } + })?; + let observed = expected + .checked_add(increment) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + return Err(CatalogRestartError::Length { + artifact, + minimum: expected, + maximum: expected, + observed, + }); + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} + Err(source) => return Err(CatalogRestartError::io(phase, source)), + } + } +} diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs new file mode 100644 index 0000000..0c53447 --- /dev/null +++ b/src/adapters/catalog_restart_loader.rs @@ -0,0 +1,105 @@ +//! This module owns capability-relative published catalog restart loading. + +use std::path::Path; + +use cap_fs_ext::DirExt; + +use crate::CatalogLength; + +use super::{ + CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + ChecksummedCatalog, ChecksummedPublicationHead, FilesystemCatalogSnapshot, catalog_restart_io, + catalog_restart_segments, physical_pool_name, +}; + +const HEAD_NAME: &str = "HEAD"; +const CATALOGS_NAME: &str = "catalogs"; +const SEGMENTS_NAME: &str = "segments"; +const HEAD_LENGTH: u64 = 128; + +pub(super) fn load( + root: &Path, + policy: CatalogRestartPolicy, +) -> Result { + let directory = catalog_restart_io::open_root(root)?; + let (head_file, observed_head_length) = catalog_restart_io::open_regular( + &directory, + HEAD_NAME, + CatalogRestartArtifact::Head, + CatalogRestartPhase::OpenHead, + )?; + if observed_head_length != HEAD_LENGTH { + return Err(CatalogRestartError::Length { + artifact: CatalogRestartArtifact::Head, + minimum: HEAD_LENGTH, + maximum: HEAD_LENGTH, + observed: observed_head_length, + }); + } + let head_bytes = catalog_restart_io::read_exact( + head_file, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadHead, + HEAD_LENGTH, + )?; + let head = ChecksummedPublicationHead::decode(&head_bytes) + .map_err(|source| CatalogRestartError::Head { source })?; + let catalogs = directory + .open_dir_nofollow(CATALOGS_NAME) + .map_err(|source| { + CatalogRestartError::io(CatalogRestartPhase::OpenCatalogDirectory, source) + })?; + let catalog_name = physical_pool_name::catalog(head.generation(), head.catalog_digest()); + let (catalog_file, observed_catalog_length) = catalog_restart_io::open_regular( + &catalogs, + &catalog_name, + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + )?; + if CatalogLength::new(observed_catalog_length).is_err() { + return Err(CatalogRestartError::Length { + artifact: CatalogRestartArtifact::Catalog, + minimum: CatalogLength::MINIMUM.get(), + maximum: CatalogLength::MAXIMUM.get(), + observed: observed_catalog_length, + }); + } + let catalog_bytes = catalog_restart_io::read_exact( + catalog_file, + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::ReadCatalog, + observed_catalog_length, + )?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes) + .map_err(|source| CatalogRestartError::Catalog { source })?; + validate_catalog_coordinate(head, catalog)?; + let segment_digests = catalog_restart_segments::collect(catalog)?; + let segments_directory = directory + .open_dir_nofollow(SEGMENTS_NAME) + .map_err(|source| { + CatalogRestartError::io(CatalogRestartPhase::OpenSegmentDirectory, source) + })?; + let segments = catalog_restart_segments::load(&segments_directory, &segment_digests, policy)?; + FilesystemCatalogSnapshot::admit(head_bytes, catalog_bytes, segments, policy) +} + +fn validate_catalog_coordinate( + head: ChecksummedPublicationHead<'_>, + catalog: ChecksummedCatalog<'_>, +) -> Result<(), CatalogRestartError> { + let generation_matches = head.generation() == catalog.generation(); + let length_matches = head.catalog_length() == catalog.length(); + let digest_matches = head.catalog_digest() == catalog.digest(); + if generation_matches && length_matches && digest_matches { + Ok(()) + } else { + Err(CatalogRestartError::CatalogCoordinate { + expected_generation: head.generation(), + observed_generation: catalog.generation(), + expected_length: head.catalog_length(), + observed_length: catalog.length(), + expected_digest: head.catalog_digest(), + observed_digest: catalog.digest(), + }) + } +} diff --git a/src/adapters/catalog_restart_phase.rs b/src/adapters/catalog_restart_phase.rs new file mode 100644 index 0000000..cf49111 --- /dev/null +++ b/src/adapters/catalog_restart_phase.rs @@ -0,0 +1,32 @@ +//! Published restart filesystem phases. + +use std::fmt; + +/// Exact filesystem operation attempted during restart loading. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogRestartPhase { + /// Pin the selected store root. + OpenRoot, + /// Open `HEAD` without following links. + OpenHead, + /// Read the complete fixed-width head. + ReadHead, + /// Open the catalog pool without following links. + OpenCatalogDirectory, + /// Open the exact head-selected catalog. + OpenCatalog, + /// Read the exact declared catalog bytes. + ReadCatalog, + /// Open the segment pool without following links. + OpenSegmentDirectory, + /// Open one exact catalog-selected segment. + OpenSegment, + /// Read one bounded complete segment. + ReadSegment, +} + +impl fmt::Display for CatalogRestartPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{self:?}") + } +} diff --git a/src/adapters/catalog_restart_policy.rs b/src/adapters/catalog_restart_policy.rs new file mode 100644 index 0000000..eb70e00 --- /dev/null +++ b/src/adapters/catalog_restart_policy.rs @@ -0,0 +1,31 @@ +//! Explicit restart-loading resource policy. + +use super::{CatalogRestartByteLimit, SegmentReadPolicy}; + +/// Bounds for one published catalog restart load. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatalogRestartPolicy { + segment_read: SegmentReadPolicy, + retained_segment_bytes: CatalogRestartByteLimit, +} + +impl CatalogRestartPolicy { + /// Creates a policy from segment grammar and aggregate retention bounds. + pub const fn new( + segment_read: SegmentReadPolicy, + retained_segment_bytes: CatalogRestartByteLimit, + ) -> Self { + Self { + segment_read, + retained_segment_bytes, + } + } + + pub(super) const fn segment_read(self) -> SegmentReadPolicy { + self.segment_read + } + + pub(super) const fn retained_segment_bytes(self) -> CatalogRestartByteLimit { + self.retained_segment_bytes + } +} diff --git a/src/adapters/catalog_restart_segments.rs b/src/adapters/catalog_restart_segments.rs new file mode 100644 index 0000000..dc89cfb --- /dev/null +++ b/src/adapters/catalog_restart_segments.rs @@ -0,0 +1,109 @@ +//! This module owns bounded loading of catalog-selected immutable segments. + +use cap_std::fs::Dir; + +use super::loaded_segment::LoadedSegment; +use super::segment_header::MAXIMUM_SEGMENT_LENGTH; +use super::{ + AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, + CatalogRestartPolicy, ChecksummedCatalog, SegmentDigest, catalog_restart_io, + physical_pool_name, +}; + +pub(super) fn collect( + catalog: ChecksummedCatalog<'_>, +) -> Result, CatalogRestartError> { + let capacity = usize::try_from(catalog.entry_count()).map_err(|_source| { + CatalogRestartError::Allocation { + artifact: CatalogRestartArtifact::Catalog, + byte_count: catalog.entry_count(), + source: None, + } + })?; + let mut digests = Vec::new(); + digests + .try_reserve_exact(capacity) + .map_err(|source| CatalogRestartError::Allocation { + artifact: CatalogRestartArtifact::Catalog, + byte_count: catalog.entry_count(), + source: Some(source), + })?; + for entry in catalog + .entries() + .map_err(|source| CatalogRestartError::Catalog { source })? + { + let entry = entry.map_err(|source| CatalogRestartError::Catalog { source })?; + digests.push(entry.segment_digest()); + } + digests.sort_unstable(); + digests.dedup(); + Ok(digests) +} + +pub(super) fn load( + directory: &Dir, + digests: &[SegmentDigest], + policy: CatalogRestartPolicy, +) -> Result, CatalogRestartError> { + let segment_count = + u64::try_from(digests.len()).map_err(|_source| CatalogRestartError::SegmentIndexLength)?; + let mut loaded = Vec::new(); + loaded.try_reserve_exact(digests.len()).map_err(|source| { + CatalogRestartError::SegmentIndexAllocation { + segment_count, + source, + } + })?; + let mut retained = 0_u64; + for digest in digests { + let artifact = CatalogRestartArtifact::Segment { digest: *digest }; + let name = physical_pool_name::segment(*digest); + let (file, observed) = catalog_restart_io::open_regular( + directory, + &name, + artifact, + CatalogRestartPhase::OpenSegment, + )?; + if observed > MAXIMUM_SEGMENT_LENGTH { + return Err(CatalogRestartError::Length { + artifact, + minimum: 0, + maximum: MAXIMUM_SEGMENT_LENGTH, + observed, + }); + } + retained = retained.checked_add(observed).ok_or( + CatalogRestartError::RetainedSegmentByteArithmetic { + current: retained, + addition: observed, + }, + )?; + if retained > policy.retained_segment_bytes().get() { + return Err(CatalogRestartError::RetainedSegmentBytes { + maximum: policy.retained_segment_bytes().get(), + observed: retained, + }); + } + let encoded = catalog_restart_io::read_exact( + file, + artifact, + CatalogRestartPhase::ReadSegment, + observed, + )?; + let segment = + AdmittedSegment::decode(&encoded, policy.segment_read()).map_err(|source| { + CatalogRestartError::Segment { + expected: *digest, + source: Box::new(source), + } + })?; + if segment.digest() != *digest { + return Err(CatalogRestartError::SegmentCoordinate { + expected: *digest, + observed: segment.digest(), + }); + } + loaded.push(LoadedSegment::new(*digest, encoded)); + } + Ok(loaded) +} diff --git a/src/adapters/filesystem_catalog_snapshot.rs b/src/adapters/filesystem_catalog_snapshot.rs new file mode 100644 index 0000000..bf68a23 --- /dev/null +++ b/src/adapters/filesystem_catalog_snapshot.rs @@ -0,0 +1,122 @@ +//! This module owns an immutable, restart-loaded filesystem snapshot. + +use std::path::Path; + +use super::loaded_segment::LoadedSegment; +use super::{ + AdmittedSegment, CatalogRestartError, CatalogRestartPolicy, CatalogSnapshot, + ChecksummedCatalog, ChecksummedPublicationHead, catalog_restart_loader, +}; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Owned bytes and proofs for one exact head-selected catalog generation. +/// +/// Loading follows only the exact catalog and segment coordinates named by the +/// checksummed publication state. Unknown files and orphaned artifacts are not +/// recovery candidates and are deliberately ignored. +#[must_use] +pub struct FilesystemCatalogSnapshot { + head_bytes: Vec, + catalog_bytes: Vec, + segments: Vec, + policy: CatalogRestartPolicy, + generation: CatalogGeneration, + catalog_digest: CatalogDigest, +} + +impl FilesystemCatalogSnapshot { + /// Loads and admits the exact snapshot selected by `HEAD`. + /// + /// The returned owner retains bounded segment bytes so every later logical + /// lookup remains pinned to this immutable generation. + /// + /// # Errors + /// + /// Returns [`CatalogRestartError`] for filesystem refusal, malformed or + /// noncanonical bytes, coordinate disagreement, resource-limit refusal, or + /// failed catalog-to-record admission. + pub fn load(root: &Path, policy: CatalogRestartPolicy) -> Result { + catalog_restart_loader::load(root, policy) + } + + /// Returns the exact generation selected during restart. + pub const fn generation(&self) -> CatalogGeneration { + self.generation + } + + /// Returns the verified digest of the selected canonical catalog. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.catalog_digest + } + + /// Reconstructs a borrowed logical snapshot from retained immutable bytes. + /// + /// Admission is repeated so no unchecked or serializer-owned state is + /// retained between the physical bytes and the logical reader view. + /// + /// # Errors + /// + /// Returns [`CatalogRestartError`] if retained bytes fail any decoder, + /// physical-coordinate, record-binding, or head-binding invariant. + pub fn snapshot(&self) -> Result, CatalogRestartError> { + let head = ChecksummedPublicationHead::decode(&self.head_bytes) + .map_err(|source| CatalogRestartError::Head { source })?; + let catalog = ChecksummedCatalog::decode(&self.catalog_bytes) + .map_err(|source| CatalogRestartError::Catalog { source })?; + let segment_count = u64::try_from(self.segments.len()) + .map_err(|_source| CatalogRestartError::SegmentIndexLength)?; + let mut segments = Vec::new(); + segments + .try_reserve_exact(self.segments.len()) + .map_err(|source| CatalogRestartError::SegmentIndexAllocation { + segment_count, + source, + })?; + for loaded in &self.segments { + let segment = AdmittedSegment::decode(loaded.encoded(), self.policy.segment_read()) + .map_err(|source| CatalogRestartError::Segment { + expected: loaded.digest(), + source: Box::new(source), + })?; + if segment.digest() != loaded.digest() { + return Err(CatalogRestartError::SegmentCoordinate { + expected: loaded.digest(), + observed: segment.digest(), + }); + } + segments.push(segment); + } + let catalog = + catalog + .admit(&segments) + .map_err(|source| CatalogRestartError::CatalogAdmission { + source: Box::new(source), + })?; + head.admit(catalog) + .map_err(|source| CatalogRestartError::Snapshot { source }) + } + + pub(super) fn admit( + head_bytes: Vec, + catalog_bytes: Vec, + segments: Vec, + policy: CatalogRestartPolicy, + ) -> Result { + let head = ChecksummedPublicationHead::decode(&head_bytes) + .map_err(|source| CatalogRestartError::Head { source })?; + let generation = head.generation(); + let catalog_digest = head.catalog_digest(); + let snapshot = Self { + head_bytes, + catalog_bytes, + segments, + policy, + generation, + catalog_digest, + }; + { + let _validated = snapshot.snapshot()?; + } + Ok(snapshot) + } +} diff --git a/src/adapters/loaded_segment.rs b/src/adapters/loaded_segment.rs new file mode 100644 index 0000000..04fe652 --- /dev/null +++ b/src/adapters/loaded_segment.rs @@ -0,0 +1,23 @@ +//! Owned bytes for one catalog-selected segment. + +use super::SegmentDigest; + +#[derive(Debug)] +pub(super) struct LoadedSegment { + digest: SegmentDigest, + encoded: Vec, +} + +impl LoadedSegment { + pub(super) const fn new(digest: SegmentDigest, encoded: Vec) -> Self { + Self { digest, encoded } + } + + pub(super) const fn digest(&self) -> SegmentDigest { + self.digest + } + + pub(super) fn encoded(&self) -> &[u8] { + &self.encoded + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 2034e05..41502d3 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -41,6 +41,14 @@ mod catalog_publication_phase; mod catalog_publication_receipt; mod catalog_publication_storage; mod catalog_record_binding; +mod catalog_restart_artifact; +mod catalog_restart_byte_limit; +mod catalog_restart_error; +mod catalog_restart_io; +mod catalog_restart_loader; +mod catalog_restart_phase; +mod catalog_restart_policy; +mod catalog_restart_segments; mod catalog_snapshot; mod catalog_snapshot_admission; mod catalog_snapshot_error; @@ -51,6 +59,7 @@ mod checksummed_catalog; mod checksummed_publication_head; mod checksummed_segment_record; mod decoded_catalog_entry; +mod filesystem_catalog_snapshot; mod filesystem_segment_stage; mod filesystem_writer_lock; mod framed_blake3; @@ -68,7 +77,9 @@ mod layout_record_decoder; mod layout_record_encoder; mod layout_record_format; mod layout_record_framing; +mod loaded_segment; mod lower_hex; +mod physical_pool_name; mod publication_head_decode_error; mod publication_head_decode_error_display; mod publication_head_decoder; @@ -152,6 +163,11 @@ pub use catalog_publication_expectation::CatalogPublicationExpectation; pub use catalog_publication_phase::CatalogPublicationPhase; pub use catalog_publication_receipt::CatalogPublicationReceipt; pub use catalog_publication_storage::CatalogPublicationStorage; +pub use catalog_restart_artifact::CatalogRestartArtifact; +pub use catalog_restart_byte_limit::{CatalogRestartByteLimit, CatalogRestartByteLimitError}; +pub use catalog_restart_error::CatalogRestartError; +pub use catalog_restart_phase::CatalogRestartPhase; +pub use catalog_restart_policy::CatalogRestartPolicy; pub use catalog_snapshot::CatalogSnapshot; pub use catalog_snapshot_error::CatalogSnapshotError; pub use catalog_successor::CatalogSuccessor; @@ -159,6 +175,7 @@ pub use catalog_transition_error::CatalogTransitionError; pub use checksummed_catalog::ChecksummedCatalog; pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; +pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_segment_stage::FilesystemSegmentStage; pub use filesystem_writer_lock::FilesystemWriterLock; pub use layout_decode_error::LayoutDecodeError; diff --git a/src/adapters/physical_pool_name.rs b/src/adapters/physical_pool_name.rs new file mode 100644 index 0000000..8828c2f --- /dev/null +++ b/src/adapters/physical_pool_name.rs @@ -0,0 +1,29 @@ +//! Exact immutable-pool filename emission. + +use std::fmt; + +use super::SegmentDigest; +use crate::{CatalogDigest, CatalogGeneration}; + +pub(super) fn segment(digest: SegmentDigest) -> String { + format!("{}.seg", DigestHex(digest.as_bytes())) +} + +pub(super) fn catalog(generation: CatalogGeneration, digest: CatalogDigest) -> String { + format!( + "{:016x}-{}.cat", + generation.get(), + DigestHex(digest.as_bytes()) + ) +} + +struct DigestHex<'a>(&'a [u8; 32]); + +impl fmt::Display for DigestHex<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} diff --git a/src/catalog/length.rs b/src/catalog/length.rs index 8e88a6e..7cf9fac 100644 --- a/src/catalog/length.rs +++ b/src/catalog/length.rs @@ -5,8 +5,8 @@ use super::CatalogLengthError; const HEADER_LENGTH: u64 = 128; const ENTRY_LENGTH: u64 = 160; const TRAILER_LENGTH: u64 = 64; -const MINIMUM: u64 = HEADER_LENGTH + TRAILER_LENGTH; -const MAXIMUM: u64 = 167_772_352; +const MINIMUM_VALUE: u64 = HEADER_LENGTH + TRAILER_LENGTH; +const MAXIMUM_VALUE: u64 = 167_772_352; /// Exact canonical byte length of one complete version-1 catalog. #[must_use] @@ -14,6 +14,12 @@ const MAXIMUM: u64 = 167_772_352; pub struct CatalogLength(u64); impl CatalogLength { + /// Smallest complete version-1 catalog length. + pub const MINIMUM: Self = Self(MINIMUM_VALUE); + + /// Largest complete version-1 catalog length. + pub const MAXIMUM: Self = Self(MAXIMUM_VALUE); + /// Admits a complete version-1 catalog length. /// /// # Errors @@ -21,17 +27,17 @@ impl CatalogLength { /// Returns [`CatalogLengthError`] when `value` exceeds the format bound or /// cannot contain a whole number of fixed-width entries. pub const fn new(value: u64) -> Result { - if value < MINIMUM || value > MAXIMUM { + if value < MINIMUM_VALUE || value > MAXIMUM_VALUE { return Err(CatalogLengthError::OutOfBounds { - minimum: MINIMUM, - maximum: MAXIMUM, + minimum: MINIMUM_VALUE, + maximum: MAXIMUM_VALUE, observed: value, }); } - let Some(entry_bytes) = value.checked_sub(MINIMUM) else { + let Some(entry_bytes) = value.checked_sub(MINIMUM_VALUE) else { return Err(CatalogLengthError::OutOfBounds { - minimum: MINIMUM, - maximum: MAXIMUM, + minimum: MINIMUM_VALUE, + maximum: MAXIMUM_VALUE, observed: value, }); }; diff --git a/src/lib.rs b/src/lib.rs index 12ecc0f..e255eb1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,18 +28,21 @@ pub use adapters::{ BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationPhase, CatalogPublicationReceipt, CatalogPublicationStorage, CatalogSnapshot, + CatalogPublicationPhase, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, - SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - WriterLockAcquireError, WriterLockAcquirePhase, publish_catalog_generation, + ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemCatalogSnapshot, + FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, + SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentReadError, SegmentReadPolicy, + SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, + SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, + SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, + SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, + SegmentWritePhase, StagedSegment, StorageProfileIdParseError, WriterLockAcquireError, + WriterLockAcquirePhase, publish_catalog_generation, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_restart.rs b/tests/catalog_restart.rs new file mode 100644 index 0000000..dd1f3ba --- /dev/null +++ b/tests/catalog_restart.rs @@ -0,0 +1,89 @@ +//! Published catalog restart-loading laws. + +#[path = "catalog_restart/refusal_laws.rs"] +mod refusal_laws; +#[path = "segment_filesystem_stage/sandbox.rs"] +pub mod sandbox; +mod support; + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use keep::{ + CatalogRestartByteLimit, CatalogRestartPolicy, ChunkId, FilesystemCatalogSnapshot, + LayoutEntryLimit, SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, +}; +use sandbox::TestDirectory; +use support::decode_hex; + +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_DIGEST: &str = "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320"; +const SEGMENT_DIGEST: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc"; +const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; + +#[test] +fn restart_reconstructs_the_exact_published_snapshot() -> Result<(), Box> { + let store = StoreFixture::create("restart-published")?; + let loaded = FilesystemCatalogSnapshot::load(store.path(), restart_policy()?)?; + let snapshot = loaded.snapshot()?; + let identity = SegmentRecordIdentity::Chunk(ChunkId::hash_bytes(&[0])?); + + assert_eq!(loaded.generation().get(), 1); + assert_eq!( + snapshot + .record(identity) + .ok_or("restart snapshot omitted its record")? + .payload(), + [0] + ); + store.remove()?; + Ok(()) +} + +struct StoreFixture { + sandbox: TestDirectory, + catalog_path: PathBuf, + segment_path: PathBuf, +} + +impl StoreFixture { + fn create(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + let catalogs = sandbox.path().join("catalogs"); + let segments = sandbox.path().join("segments"); + fs::create_dir(&catalogs)?; + fs::create_dir(&segments)?; + fs::write(sandbox.path().join("HEAD"), fixture(HEAD_HEX)?)?; + let catalog_path = catalogs.join(format!("0000000000000001-{CATALOG_DIGEST}.cat")); + let segment_path = segments.join(format!("{SEGMENT_DIGEST}.seg")); + fs::write(&catalog_path, fixture(CATALOG_HEX)?)?; + fs::write(&segment_path, fixture(SEGMENT_HEX)?)?; + Ok(Self { + sandbox, + catalog_path, + segment_path, + }) + } + + fn path(&self) -> &Path { + self.sandbox.path() + } + + fn remove(self) -> Result<(), Box> { + self.sandbox.remove().map_err(Into::into) + } +} + +fn restart_policy() -> Result> { + Ok(CatalogRestartPolicy::new( + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), + CatalogRestartByteLimit::new(RETAINED_SEGMENT_LIMIT)?, + )) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/catalog_restart/refusal_laws.rs b/tests/catalog_restart/refusal_laws.rs new file mode 100644 index 0000000..9afdc9e --- /dev/null +++ b/tests/catalog_restart/refusal_laws.rs @@ -0,0 +1,139 @@ +//! Published restart corruption, dangling-state, and conflict laws. + +use std::error::Error; +use std::fs; +use std::io::ErrorKind; + +use keep::{ + CatalogDecodeError, CatalogRestartError, CatalogRestartPhase, FilesystemCatalogSnapshot, + PublicationHeadDecodeError, +}; + +use super::{StoreFixture, fixture, restart_policy}; +use crate::support::require_error; + +const BUNDLE_CATALOG_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/empty-segment.hex"); +const HEAD_VERSION_OFFSET: usize = 17; +const CATALOG_FLAGS_OFFSET: usize = 19; + +#[test] +fn corrupt_and_unsupported_heads_refuse_at_the_head_boundary() -> Result<(), Box> { + let corrupt = StoreFixture::create("restart-corrupt-head")?; + let mut bytes = fs::read(corrupt.path().join("HEAD"))?; + *bytes.last_mut().ok_or("head fixture is empty")? ^= 1; + fs::write(corrupt.path().join("HEAD"), bytes)?; + let error = require_error( + FilesystemCatalogSnapshot::load(corrupt.path(), restart_policy()?), + "corrupt head was loaded", + )?; + assert!(matches!( + error, + CatalogRestartError::Head { + source: PublicationHeadDecodeError::ChecksumMismatch { .. } + } + )); + corrupt.remove()?; + + let unsupported = StoreFixture::create("restart-unsupported-head")?; + let mut bytes = fs::read(unsupported.path().join("HEAD"))?; + *bytes + .get_mut(HEAD_VERSION_OFFSET) + .ok_or("head lacks version field")? = 2; + fs::write(unsupported.path().join("HEAD"), bytes)?; + let error = require_error( + FilesystemCatalogSnapshot::load(unsupported.path(), restart_policy()?), + "unsupported head was loaded", + )?; + assert!(matches!( + error, + CatalogRestartError::Head { + source: PublicationHeadDecodeError::UnsupportedVersion { observed: 2, .. } + } + )); + unsupported.remove() +} + +#[test] +fn noncanonical_catalog_refuses_before_segment_loading() -> Result<(), Box> { + let store = StoreFixture::create("restart-noncanonical-catalog")?; + let mut bytes = fs::read(&store.catalog_path)?; + *bytes + .get_mut(CATALOG_FLAGS_OFFSET) + .ok_or("catalog lacks flags field")? = 1; + fs::write(&store.catalog_path, bytes)?; + let error = require_error( + FilesystemCatalogSnapshot::load(store.path(), restart_policy()?), + "noncanonical catalog was loaded", + )?; + + assert!(matches!( + error, + CatalogRestartError::Catalog { + source: CatalogDecodeError::Flags { observed: 1, .. } + } + )); + store.remove() +} + +#[test] +fn dangling_catalog_and_segment_paths_refuse_exactly() -> Result<(), Box> { + let missing_catalog = StoreFixture::create("restart-missing-catalog")?; + fs::remove_file(&missing_catalog.catalog_path)?; + let error = require_error( + FilesystemCatalogSnapshot::load(missing_catalog.path(), restart_policy()?), + "missing catalog was loaded", + )?; + assert!(matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenCatalog, + ref source, + } if source.kind() == ErrorKind::NotFound + )); + missing_catalog.remove()?; + + let missing_segment = StoreFixture::create("restart-missing-segment")?; + fs::remove_file(&missing_segment.segment_path)?; + let error = require_error( + FilesystemCatalogSnapshot::load(missing_segment.path(), restart_policy()?), + "missing segment was loaded", + )?; + assert!(matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenSegment, + ref source, + } if source.kind() == ErrorKind::NotFound + )); + missing_segment.remove() +} + +#[test] +fn physical_name_content_conflicts_are_never_substituted() -> Result<(), Box> { + let catalog = StoreFixture::create("restart-conflicting-catalog")?; + fs::write(&catalog.catalog_path, fixture(BUNDLE_CATALOG_HEX)?)?; + let error = require_error( + FilesystemCatalogSnapshot::load(catalog.path(), restart_policy()?), + "wrong catalog bytes were substituted under the selected name", + )?; + assert!(matches!( + error, + CatalogRestartError::CatalogCoordinate { .. } + )); + catalog.remove()?; + + let segment = StoreFixture::create("restart-conflicting-segment")?; + fs::write(&segment.segment_path, fixture(EMPTY_SEGMENT_HEX)?)?; + let error = require_error( + FilesystemCatalogSnapshot::load(segment.path(), restart_policy()?), + "wrong segment bytes were substituted under the selected name", + )?; + assert!(matches!( + error, + CatalogRestartError::SegmentCoordinate { .. } + )); + segment.remove() +} From 67f6b73ae838ecbeb03537877a8e17b8476b459d Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:21:31 -0700 Subject: [PATCH 12/31] Publish catalogs through the filesystem --- src/adapters/catalog_restart_loader.rs | 12 +- src/adapters/filesystem_catalog_artifact.rs | 152 ++++++++++++++++++ src/adapters/filesystem_catalog_catalog.rs | 89 ++++++++++ src/adapters/filesystem_catalog_current.rs | 65 ++++++++ src/adapters/filesystem_catalog_head.rs | 93 +++++++++++ .../filesystem_catalog_publication_error.rs | 54 +++++++ src/adapters/filesystem_catalog_publisher.rs | 56 +++++++ src/adapters/filesystem_catalog_segment.rs | 52 ++++++ src/adapters/filesystem_catalog_snapshot.rs | 4 + src/adapters/filesystem_catalog_storage.rs | 104 ++++++++++++ src/adapters/filesystem_writer_lock.rs | 8 +- src/adapters/mod.rs | 10 ++ src/lib.rs | 22 +-- tests/catalog_filesystem_publication.rs | 119 ++++++++++++++ .../refusal_laws.rs | 142 ++++++++++++++++ 15 files changed, 967 insertions(+), 15 deletions(-) create mode 100644 src/adapters/filesystem_catalog_artifact.rs create mode 100644 src/adapters/filesystem_catalog_catalog.rs create mode 100644 src/adapters/filesystem_catalog_current.rs create mode 100644 src/adapters/filesystem_catalog_head.rs create mode 100644 src/adapters/filesystem_catalog_publication_error.rs create mode 100644 src/adapters/filesystem_catalog_publisher.rs create mode 100644 src/adapters/filesystem_catalog_segment.rs create mode 100644 src/adapters/filesystem_catalog_storage.rs create mode 100644 tests/catalog_filesystem_publication.rs create mode 100644 tests/catalog_filesystem_publication/refusal_laws.rs diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs index 0c53447..ce96d44 100644 --- a/src/adapters/catalog_restart_loader.rs +++ b/src/adapters/catalog_restart_loader.rs @@ -22,9 +22,17 @@ pub(super) fn load( policy: CatalogRestartPolicy, ) -> Result { let directory = catalog_restart_io::open_root(root)?; + load_from_directory(&directory, HEAD_NAME, policy) +} + +pub(super) fn load_from_directory( + directory: &cap_std::fs::Dir, + head_name: &str, + policy: CatalogRestartPolicy, +) -> Result { let (head_file, observed_head_length) = catalog_restart_io::open_regular( - &directory, - HEAD_NAME, + directory, + head_name, CatalogRestartArtifact::Head, CatalogRestartPhase::OpenHead, )?; diff --git a/src/adapters/filesystem_catalog_artifact.rs b/src/adapters/filesystem_catalog_artifact.rs new file mode 100644 index 0000000..2c9e7db --- /dev/null +++ b/src/adapters/filesystem_catalog_artifact.rs @@ -0,0 +1,152 @@ +//! This module owns exact staged and pooled publication artifact operations. + +use std::error::Error; +use std::io; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, File, OpenOptions}; + +use super::segment_header::MAXIMUM_SEGMENT_LENGTH; +use super::{ + AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, + ChecksummedCatalog, FilesystemCatalogPublicationError, SegmentReadPolicy, catalog_restart_io, +}; +use crate::CatalogLength; + +pub(super) fn create_exclusive(directory: &Dir, name: &str) -> io::Result { + let mut options = OpenOptions::new(); + options + .write(true) + .create_new(true) + .follow(FollowSymlinks::No) + .nonblock(true); + directory.open_with(name, &options) +} + +pub(super) fn synchronize_directory(directory: &Dir) -> io::Result<()> { + directory.try_clone()?.into_std_file().sync_all() +} + +pub(super) fn link_without_replacement( + source_directory: &Dir, + source_name: &str, + destination_directory: &Dir, + destination_name: &str, +) -> io::Result<()> { + match source_directory.hard_link(source_name, destination_directory, destination_name) { + Ok(()) => Ok(()), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => Ok(()), + Err(source) => Err(source), + } +} + +pub(super) fn verify_segment( + directory: &Dir, + name: &str, + expected: &AdmittedSegment<'_>, + policy: SegmentReadPolicy, +) -> io::Result<()> { + let artifact = CatalogRestartArtifact::Segment { + digest: expected.digest(), + }; + let (file, observed) = catalog_restart_io::open_regular( + directory, + name, + artifact, + CatalogRestartPhase::OpenSegment, + ) + .map_err(invalid_data)?; + if observed > MAXIMUM_SEGMENT_LENGTH { + return Err(invalid_data(CatalogRestartError::Length { + artifact, + minimum: 0, + maximum: MAXIMUM_SEGMENT_LENGTH, + observed, + })); + } + let encoded = + catalog_restart_io::read_exact(file, artifact, CatalogRestartPhase::ReadSegment, observed) + .map_err(invalid_data)?; + let decoded = AdmittedSegment::decode(&encoded, policy).map_err(|source| { + invalid_data(CatalogRestartError::Segment { + expected: expected.digest(), + source: Box::new(source), + }) + })?; + if decoded.digest() != expected.digest() { + return Err(invalid_data(CatalogRestartError::SegmentCoordinate { + expected: expected.digest(), + observed: decoded.digest(), + })); + } + require_exact_bytes(artifact, &encoded, expected.encoded()) +} + +pub(super) fn verify_catalog( + directory: &Dir, + name: &str, + expected: ChecksummedCatalog<'_>, +) -> io::Result<()> { + let artifact = CatalogRestartArtifact::Catalog; + let (file, observed) = catalog_restart_io::open_regular( + directory, + name, + artifact, + CatalogRestartPhase::OpenCatalog, + ) + .map_err(invalid_data)?; + if CatalogLength::new(observed).is_err() { + return Err(invalid_data(CatalogRestartError::Length { + artifact, + minimum: CatalogLength::MINIMUM.get(), + maximum: CatalogLength::MAXIMUM.get(), + observed, + })); + } + let encoded = + catalog_restart_io::read_exact(file, artifact, CatalogRestartPhase::ReadCatalog, observed) + .map_err(invalid_data)?; + let decoded = ChecksummedCatalog::decode(&encoded) + .map_err(|source| invalid_data(CatalogRestartError::Catalog { source }))?; + require_catalog_coordinate(expected, decoded)?; + require_exact_bytes(artifact, &encoded, expected.encoded()) +} + +pub(super) fn invalid_data(source: impl Error + Send + Sync + 'static) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, source) +} + +fn require_catalog_coordinate( + expected: ChecksummedCatalog<'_>, + observed: ChecksummedCatalog<'_>, +) -> io::Result<()> { + let generation_matches = expected.generation() == observed.generation(); + let length_matches = expected.length() == observed.length(); + let digest_matches = expected.digest() == observed.digest(); + if generation_matches && length_matches && digest_matches { + Ok(()) + } else { + Err(invalid_data(CatalogRestartError::CatalogCoordinate { + expected_generation: expected.generation(), + observed_generation: observed.generation(), + expected_length: expected.length(), + observed_length: observed.length(), + expected_digest: expected.digest(), + observed_digest: observed.digest(), + })) + } +} + +fn require_exact_bytes( + artifact: CatalogRestartArtifact, + observed: &[u8], + expected: &[u8], +) -> io::Result<()> { + if observed == expected { + Ok(()) + } else { + Err(invalid_data( + FilesystemCatalogPublicationError::ByteConflict { artifact }, + )) + } +} diff --git a/src/adapters/filesystem_catalog_catalog.rs b/src/adapters/filesystem_catalog_catalog.rs new file mode 100644 index 0000000..5648a2c --- /dev/null +++ b/src/adapters/filesystem_catalog_catalog.rs @@ -0,0 +1,89 @@ +//! This module owns filesystem catalog-stage and pool transitions. + +use std::io::{self, Write}; + +use super::{ + CanonicalCatalog, CatalogRestartArtifact, ChecksummedCatalog, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, filesystem_catalog_artifact, + filesystem_catalog_publisher, physical_pool_name, +}; + +pub(super) fn create_stage(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { + if publisher.catalog_stage.is_some() { + return Err(stage_state()); + } + publisher.catalog_stage = Some(filesystem_catalog_artifact::create_exclusive( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_CATALOG, + )?); + Ok(()) +} + +pub(super) fn write( + publisher: &mut FilesystemCatalogPublisher, + catalog: &CanonicalCatalog, +) -> io::Result<()> { + stage_mut(publisher)?.write_all(catalog.encoded()) +} + +pub(super) fn flush(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { + stage_mut(publisher)?.flush() +} + +pub(super) fn synchronize(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { + stage_mut(publisher)?.sync_all() +} + +pub(super) fn link( + publisher: &mut FilesystemCatalogPublisher, + catalog: ChecksummedCatalog<'_>, +) -> io::Result<()> { + let stage = publisher.catalog_stage.take().ok_or_else(stage_state)?; + drop(stage); + filesystem_catalog_artifact::verify_catalog( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_CATALOG, + catalog, + )?; + filesystem_catalog_artifact::link_without_replacement( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_CATALOG, + &publisher.catalogs, + &physical_pool_name::catalog(catalog.generation(), catalog.digest()), + ) +} + +pub(super) fn verify_pool( + publisher: &FilesystemCatalogPublisher, + catalog: ChecksummedCatalog<'_>, +) -> io::Result<()> { + filesystem_catalog_artifact::verify_catalog( + &publisher.catalogs, + &physical_pool_name::catalog(catalog.generation(), catalog.digest()), + catalog, + ) +} + +pub(super) fn synchronize_pool(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&publisher.catalogs) +} + +pub(super) fn remove_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + publisher + .staging + .remove_file(filesystem_catalog_publisher::CURRENT_CATALOG) +} + +pub(super) fn synchronize_staging(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&publisher.staging) +} + +fn stage_mut(publisher: &mut FilesystemCatalogPublisher) -> io::Result<&mut cap_std::fs::File> { + publisher.catalog_stage.as_mut().ok_or_else(stage_state) +} + +fn stage_state() -> io::Error { + filesystem_catalog_artifact::invalid_data(FilesystemCatalogPublicationError::StageState { + artifact: CatalogRestartArtifact::Catalog, + }) +} diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs new file mode 100644 index 0000000..ed92869 --- /dev/null +++ b/src/adapters/filesystem_catalog_current.rs @@ -0,0 +1,65 @@ +//! This module owns writer-locked current-head verification. + +use std::io; + +use super::{ + CatalogPublicationExpectation, CatalogRestartError, CatalogRestartPhase, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, catalog_restart_loader, + filesystem_catalog_artifact, +}; + +pub(super) fn verify( + publisher: &FilesystemCatalogPublisher, + expected: CatalogPublicationExpectation, +) -> io::Result<()> { + require_no_next_head(publisher)?; + match catalog_restart_loader::load_from_directory( + &publisher.root, + super::filesystem_catalog_publisher::HEAD, + publisher.policy, + ) { + Ok(observed) => { + let observed_generation = Some(observed.generation()); + let observed_digest = Some(observed.catalog_digest()); + if expected.current_generation() == observed_generation + && expected.current_catalog_digest() == observed_digest + { + Ok(()) + } else { + Err(filesystem_catalog_artifact::invalid_data( + FilesystemCatalogPublicationError::CurrentState { + expected_generation: expected.current_generation(), + expected_digest: expected.current_catalog_digest(), + observed_generation, + observed_digest, + }, + )) + } + } + Err(source) if head_is_absent(&source) && expected.current_generation().is_none() => Ok(()), + Err(source) => Err(filesystem_catalog_artifact::invalid_data(source)), + } +} + +fn require_no_next_head(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + match publisher + .root + .symlink_metadata(super::filesystem_catalog_publisher::NEXT_HEAD) + { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(source), + Ok(_metadata) => Err(filesystem_catalog_artifact::invalid_data( + FilesystemCatalogPublicationError::HeadRecoveryRequired, + )), + } +} + +fn head_is_absent(error: &CatalogRestartError) -> bool { + matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenHead, + source, + } if source.kind() == io::ErrorKind::NotFound + ) +} diff --git a/src/adapters/filesystem_catalog_head.rs b/src/adapters/filesystem_catalog_head.rs new file mode 100644 index 0000000..1c465e8 --- /dev/null +++ b/src/adapters/filesystem_catalog_head.rs @@ -0,0 +1,93 @@ +//! This module owns filesystem publication-head transitions. + +use std::io::{self, Write}; + +use super::{ + CanonicalPublicationHead, CatalogRestartArtifact, CatalogSnapshot, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, catalog_restart_loader, + filesystem_catalog_artifact, filesystem_catalog_publisher, +}; + +pub(super) fn create_stage(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { + if publisher.head_stage.is_some() { + return Err(stage_state()); + } + publisher.head_stage = Some(filesystem_catalog_artifact::create_exclusive( + &publisher.root, + filesystem_catalog_publisher::NEXT_HEAD, + )?); + Ok(()) +} + +pub(super) fn write( + publisher: &mut FilesystemCatalogPublisher, + head: &CanonicalPublicationHead, +) -> io::Result<()> { + stage_mut(publisher)?.write_all(head.encoded()) +} + +pub(super) fn flush(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { + stage_mut(publisher)?.flush() +} + +pub(super) fn synchronize(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { + stage_mut(publisher)?.sync_all() +} + +pub(super) fn verify_view( + publisher: &mut FilesystemCatalogPublisher, + head: &CanonicalPublicationHead, + expected: &CatalogSnapshot<'_, '_, '_>, +) -> io::Result<()> { + let stage = publisher.head_stage.take().ok_or_else(stage_state)?; + drop(stage); + let observed = catalog_restart_loader::load_from_directory( + &publisher.root, + filesystem_catalog_publisher::NEXT_HEAD, + publisher.policy, + ) + .map_err(filesystem_catalog_artifact::invalid_data)?; + if observed.head_bytes() != head.encoded() { + return Err(filesystem_catalog_artifact::invalid_data( + FilesystemCatalogPublicationError::ByteConflict { + artifact: CatalogRestartArtifact::Head, + }, + )); + } + if observed.generation() == expected.generation() + && observed.catalog_digest() == expected.catalog_digest() + { + Ok(()) + } else { + Err(filesystem_catalog_artifact::invalid_data( + FilesystemCatalogPublicationError::CurrentState { + expected_generation: Some(expected.generation()), + expected_digest: Some(expected.catalog_digest()), + observed_generation: Some(observed.generation()), + observed_digest: Some(observed.catalog_digest()), + }, + )) + } +} + +pub(super) fn replace(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + publisher.root.rename( + filesystem_catalog_publisher::NEXT_HEAD, + &publisher.root, + filesystem_catalog_publisher::HEAD, + ) +} + +pub(super) fn synchronize_root(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&publisher.root) +} + +fn stage_mut(publisher: &mut FilesystemCatalogPublisher) -> io::Result<&mut cap_std::fs::File> { + publisher.head_stage.as_mut().ok_or_else(stage_state) +} + +fn stage_state() -> io::Error { + filesystem_catalog_artifact::invalid_data(FilesystemCatalogPublicationError::StageState { + artifact: CatalogRestartArtifact::Head, + }) +} diff --git a/src/adapters/filesystem_catalog_publication_error.rs b/src/adapters/filesystem_catalog_publication_error.rs new file mode 100644 index 0000000..898c51c --- /dev/null +++ b/src/adapters/filesystem_catalog_publication_error.rs @@ -0,0 +1,54 @@ +//! This module owns filesystem catalog publication invariant failures. + +use std::error::Error; +use std::fmt; + +use super::CatalogRestartArtifact; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Filesystem state disagreed with a preflighted publication invariant. +#[derive(Debug)] +pub enum FilesystemCatalogPublicationError { + /// The current publication coordinate was stale or unexpectedly present. + CurrentState { + /// Generation required by the caller, absent for initialization. + expected_generation: Option, + /// Digest required by the caller, absent for initialization. + expected_digest: Option, + /// Generation verified from `HEAD`, absent when no head exists. + observed_generation: Option, + /// Digest verified from `HEAD`, absent when no head exists. + observed_digest: Option, + }, + /// A stage handle required by the current phase was absent. + StageState { + /// Artifact whose writable stage was not retained. + artifact: CatalogRestartArtifact, + }, + /// Valid bytes at a physical coordinate differed from preflighted bytes. + ByteConflict { + /// Artifact whose exact bytes disagreed. + artifact: CatalogRestartArtifact, + }, + /// A leftover `head.next` requires explicit recovery. + HeadRecoveryRequired, +} + +impl fmt::Display for FilesystemCatalogPublicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentState { .. } => { + formatter.write_str("current catalog publication state is stale") + } + Self::StageState { .. } => { + formatter.write_str("catalog publication stage state is invalid") + } + Self::ByteConflict { .. } => formatter.write_str("publication artifact bytes conflict"), + Self::HeadRecoveryRequired => { + formatter.write_str("head.next requires explicit recovery") + } + } + } +} + +impl Error for FilesystemCatalogPublicationError {} diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs new file mode 100644 index 0000000..58fe8ba --- /dev/null +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -0,0 +1,56 @@ +//! This module owns writer-locked filesystem catalog publication state. + +use std::io; + +use cap_fs_ext::DirExt; +use cap_std::fs::{Dir, File}; + +use super::{CatalogRestartPolicy, FilesystemWriterLock}; + +pub(super) const CURRENT_SEGMENT: &str = "current.seg"; +pub(super) const CURRENT_CATALOG: &str = "current.cat"; +pub(super) const HEAD: &str = "HEAD"; +pub(super) const NEXT_HEAD: &str = "head.next"; + +/// Exclusive filesystem authority for one catalog publication at a time. +/// +/// The publisher owns the writer lock and pinned root, staging, segment-pool, +/// and catalog-pool directory capabilities until it is dropped. Dropping it +/// closes open stages and releases the writer lock but never publishes, +/// removes, truncates, or repairs protocol state. +#[must_use] +pub struct FilesystemCatalogPublisher { + pub(super) _lock: FilesystemWriterLock, + pub(super) root: Dir, + pub(super) staging: Dir, + pub(super) segments: Dir, + pub(super) catalogs: Dir, + pub(super) policy: CatalogRestartPolicy, + pub(super) catalog_stage: Option, + pub(super) head_stage: Option, +} + +impl FilesystemCatalogPublisher { + /// Pins the canonical publication directories under an acquired writer lock. + /// + /// # Errors + /// + /// Returns the exact root-clone or no-follow directory-open failure. A + /// failure drops `lock` and therefore releases writer authority. + pub fn open(lock: FilesystemWriterLock, policy: CatalogRestartPolicy) -> io::Result { + let root = lock.clone_directory()?; + let staging = root.open_dir_nofollow("staging")?; + let segments = root.open_dir_nofollow("segments")?; + let catalogs = root.open_dir_nofollow("catalogs")?; + Ok(Self { + _lock: lock, + root, + staging, + segments, + catalogs, + policy, + catalog_stage: None, + head_stage: None, + }) + } +} diff --git a/src/adapters/filesystem_catalog_segment.rs b/src/adapters/filesystem_catalog_segment.rs new file mode 100644 index 0000000..d98d912 --- /dev/null +++ b/src/adapters/filesystem_catalog_segment.rs @@ -0,0 +1,52 @@ +//! This module owns filesystem segment-pool publication transitions. + +use std::io; + +use super::{ + AdmittedSegment, FilesystemCatalogPublisher, filesystem_catalog_artifact, + filesystem_catalog_publisher, physical_pool_name, +}; + +pub(super) fn link( + publisher: &FilesystemCatalogPublisher, + segment: &AdmittedSegment<'_>, +) -> io::Result<()> { + filesystem_catalog_artifact::verify_segment( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_SEGMENT, + segment, + publisher.policy.segment_read(), + )?; + filesystem_catalog_artifact::link_without_replacement( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_SEGMENT, + &publisher.segments, + &physical_pool_name::segment(segment.digest()), + ) +} + +pub(super) fn verify_pool( + publisher: &FilesystemCatalogPublisher, + segment: &AdmittedSegment<'_>, +) -> io::Result<()> { + filesystem_catalog_artifact::verify_segment( + &publisher.segments, + &physical_pool_name::segment(segment.digest()), + segment, + publisher.policy.segment_read(), + ) +} + +pub(super) fn synchronize_pool(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&publisher.segments) +} + +pub(super) fn remove_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + publisher + .staging + .remove_file(filesystem_catalog_publisher::CURRENT_SEGMENT) +} + +pub(super) fn synchronize_staging(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&publisher.staging) +} diff --git a/src/adapters/filesystem_catalog_snapshot.rs b/src/adapters/filesystem_catalog_snapshot.rs index bf68a23..d90ef6d 100644 --- a/src/adapters/filesystem_catalog_snapshot.rs +++ b/src/adapters/filesystem_catalog_snapshot.rs @@ -119,4 +119,8 @@ impl FilesystemCatalogSnapshot { } Ok(snapshot) } + + pub(super) fn head_bytes(&self) -> &[u8] { + &self.head_bytes + } } diff --git a/src/adapters/filesystem_catalog_storage.rs b/src/adapters/filesystem_catalog_storage.rs new file mode 100644 index 0000000..80f8385 --- /dev/null +++ b/src/adapters/filesystem_catalog_storage.rs @@ -0,0 +1,104 @@ +//! This module binds filesystem catalog transitions to the publication port. + +use std::io; + +use super::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, FilesystemCatalogPublisher, + filesystem_catalog_catalog, filesystem_catalog_current, filesystem_catalog_head, + filesystem_catalog_segment, +}; + +impl CatalogPublicationStorage for FilesystemCatalogPublisher { + fn verify_current(&mut self, expected: CatalogPublicationExpectation) -> io::Result<()> { + filesystem_catalog_current::verify(self, expected) + } + + fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + filesystem_catalog_segment::link(self, segment) + } + + fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + filesystem_catalog_segment::verify_pool(self, segment) + } + + fn synchronize_segments(&mut self) -> io::Result<()> { + filesystem_catalog_segment::synchronize_pool(self) + } + + fn remove_segment_stage(&mut self) -> io::Result<()> { + filesystem_catalog_segment::remove_stage(self) + } + + fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { + filesystem_catalog_segment::synchronize_staging(self) + } + + fn create_catalog_stage(&mut self) -> io::Result<()> { + filesystem_catalog_catalog::create_stage(self) + } + + fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()> { + filesystem_catalog_catalog::write(self, catalog) + } + + fn flush_catalog(&mut self) -> io::Result<()> { + filesystem_catalog_catalog::flush(self) + } + + fn synchronize_catalog(&mut self) -> io::Result<()> { + filesystem_catalog_catalog::synchronize(self) + } + + fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + filesystem_catalog_catalog::link(self, catalog) + } + + fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + filesystem_catalog_catalog::verify_pool(self, catalog) + } + + fn synchronize_catalogs(&mut self) -> io::Result<()> { + filesystem_catalog_catalog::synchronize_pool(self) + } + + fn remove_catalog_stage(&mut self) -> io::Result<()> { + filesystem_catalog_catalog::remove_stage(self) + } + + fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { + filesystem_catalog_catalog::synchronize_staging(self) + } + + fn create_head_stage(&mut self) -> io::Result<()> { + filesystem_catalog_head::create_stage(self) + } + + fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()> { + filesystem_catalog_head::write(self, head) + } + + fn flush_head(&mut self) -> io::Result<()> { + filesystem_catalog_head::flush(self) + } + + fn synchronize_head(&mut self) -> io::Result<()> { + filesystem_catalog_head::synchronize(self) + } + + fn verify_head_view( + &mut self, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result<()> { + filesystem_catalog_head::verify_view(self, head, snapshot) + } + + fn replace_head(&mut self) -> io::Result<()> { + filesystem_catalog_head::replace(self) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + filesystem_catalog_head::synchronize_root(self) + } +} diff --git a/src/adapters/filesystem_writer_lock.rs b/src/adapters/filesystem_writer_lock.rs index 048bbe3..fc02da6 100644 --- a/src/adapters/filesystem_writer_lock.rs +++ b/src/adapters/filesystem_writer_lock.rs @@ -18,7 +18,7 @@ const LOCK_FILE_NAME: &str = "writer.lock"; /// it never deletes, renames, truncates, or replaces `writer.lock`. #[must_use] pub struct FilesystemWriterLock { - _directory: Dir, + directory: Dir, _lock_file: File, } @@ -59,7 +59,7 @@ impl FilesystemWriterLock { let lock_file = lock_file.into_std(); match lock_file.try_lock() { Ok(()) => Ok(Self { - _directory: directory, + directory, _lock_file: lock_file, }), Err(TryLockError::WouldBlock) => Err(WriterLockAcquireError::Busy), @@ -69,4 +69,8 @@ impl FilesystemWriterLock { )), } } + + pub(super) fn clone_directory(&self) -> std::io::Result { + self.directory.try_clone() + } } diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 41502d3..be33c3d 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -59,7 +59,15 @@ mod checksummed_catalog; mod checksummed_publication_head; mod checksummed_segment_record; mod decoded_catalog_entry; +mod filesystem_catalog_artifact; +mod filesystem_catalog_catalog; +mod filesystem_catalog_current; +mod filesystem_catalog_head; +mod filesystem_catalog_publication_error; +mod filesystem_catalog_publisher; +mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; +mod filesystem_catalog_storage; mod filesystem_segment_stage; mod filesystem_writer_lock; mod framed_blake3; @@ -175,6 +183,8 @@ pub use catalog_transition_error::CatalogTransitionError; pub use checksummed_catalog::ChecksummedCatalog; pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; +pub use filesystem_catalog_publication_error::FilesystemCatalogPublicationError; +pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; pub use filesystem_segment_stage::FilesystemSegmentStage; pub use filesystem_writer_lock::FilesystemWriterLock; diff --git a/src/lib.rs b/src/lib.rs index e255eb1..a6f6152 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,17 +32,17 @@ pub use adapters::{ CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemCatalogSnapshot, - FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, - SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentReadError, SegmentReadPolicy, - SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, - SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, - SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, - SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, - SegmentWritePhase, StagedSegment, StorageProfileIdParseError, WriterLockAcquireError, - WriterLockAcquirePhase, publish_catalog_generation, + ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + WriterLockAcquireError, WriterLockAcquirePhase, publish_catalog_generation, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs new file mode 100644 index 0000000..f4fffd1 --- /dev/null +++ b/tests/catalog_filesystem_publication.rs @@ -0,0 +1,119 @@ +//! Filesystem-backed catalog publication laws. + +#[path = "catalog_filesystem_publication/refusal_laws.rs"] +mod refusal_laws; +#[path = "segment_filesystem_stage/sandbox.rs"] +pub mod sandbox; +mod support; + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, + FilesystemCatalogSnapshot, FilesystemWriterLock, LayoutEntryLimit, SegmentPublication, + SegmentReadPolicy, SegmentRecordLimit, publish_catalog_generation, +}; +use sandbox::TestDirectory; +use support::decode_hex; + +const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/empty-segment.hex"); +const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_DIGEST: &str = "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320"; +const SEGMENT_DIGEST: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc"; +const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; + +#[test] +fn successful_publication_materializes_only_the_exact_durable_view() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-success")?; + let segment_bytes = fixture(SEGMENT_HEX)?; + fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + + let receipt = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(&segments[0]), + &catalog, + &segments, + )?; + drop(publisher); + + assert_eq!(receipt.generation().get(), 1); + assert_eq!(fs::read(store.path().join("HEAD"))?, fixture(HEAD_HEX)?); + assert_eq!(fs::read(store.catalog_path())?, fixture(CATALOG_HEX)?); + assert_eq!(fs::read(store.segment_path())?, segment_bytes); + assert!(!store.staging().join("current.seg").exists()); + assert!(!store.staging().join("current.cat").exists()); + assert!(!store.path().join("head.next").exists()); + let loaded = FilesystemCatalogSnapshot::load(store.path(), restart_policy()?)?; + assert_eq!(loaded.catalog_digest(), receipt.catalog_digest()); + store.remove() +} + +struct StoreFixture { + sandbox: TestDirectory, + catalog_path: PathBuf, + segment_path: PathBuf, +} + +impl StoreFixture { + fn create(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::write(sandbox.path().join("writer.lock"), [])?; + let staging = sandbox.path().join("staging"); + let segments = sandbox.path().join("segments"); + let catalogs = sandbox.path().join("catalogs"); + fs::create_dir(&staging)?; + fs::create_dir(&segments)?; + fs::create_dir(&catalogs)?; + Ok(Self { + catalog_path: catalogs.join(format!("0000000000000001-{CATALOG_DIGEST}.cat")), + segment_path: segments.join(format!("{SEGMENT_DIGEST}.seg")), + sandbox, + }) + } + + fn path(&self) -> &Path { + self.sandbox.path() + } + + fn staging(&self) -> PathBuf { + self.path().join("staging") + } + + fn catalog_path(&self) -> &Path { + &self.catalog_path + } + + fn segment_path(&self) -> &Path { + &self.segment_path + } + + fn remove(self) -> Result<(), Box> { + self.sandbox.remove().map_err(Into::into) + } +} + +fn restart_policy() -> Result> { + Ok(CatalogRestartPolicy::new( + maximum_segment_policy(), + CatalogRestartByteLimit::new(RETAINED_SEGMENT_LIMIT)?, + )) +} + +const fn maximum_segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs new file mode 100644 index 0000000..f43ae1a --- /dev/null +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -0,0 +1,142 @@ +//! Filesystem publication conflict, staleness, and recovery-refusal laws. + +use std::error::Error; +use std::fs; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationPhase, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemWriterLock, SegmentPublication, + publish_catalog_generation, +}; + +use super::{ + EMPTY_SEGMENT_HEX, SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, +}; +use crate::support::require_error; + +#[test] +fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-conflict")?; + let segment_bytes = fixture(SEGMENT_HEX)?; + fs::write(store.staging().join("current.seg"), &segment_bytes)?; + fs::write(store.segment_path(), fixture(EMPTY_SEGMENT_HEX)?)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + + let error = require_error( + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(&segments[0]), + &catalog, + &segments, + ), + "conflicting immutable segment was published", + )?; + drop(publisher); + + assert!(matches!( + error, + CatalogPublicationError::Storage { + phase: CatalogPublicationPhase::VerifySegmentPool, + .. + } + )); + assert!(!store.path().join("HEAD").exists()); + assert!(store.staging().join("current.seg").exists()); + store.remove() +} + +#[test] +fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-stale")?; + let segment_bytes = fixture(SEGMENT_HEX)?; + fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let _receipt = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(&segments[0]), + &catalog, + &segments, + )?; + drop(publisher); + + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let error = require_error( + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::None, + &catalog, + &segments, + ), + "stale uninitialized expectation was accepted", + )?; + + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("stale expectation returned the wrong error".into()); + }; + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert!(matches!( + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(FilesystemCatalogPublicationError::CurrentState { + expected_generation: None, + observed_generation: Some(_), + .. + }) + )); + drop(publisher); + assert!(!store.staging().join("current.cat").exists()); + store.remove() +} + +#[test] +fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-next-head")?; + fs::write(store.path().join("head.next"), [])?; + let segment_bytes = fixture(SEGMENT_HEX)?; + fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + + let error = require_error( + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::One(&segments[0]), + &catalog, + &segments, + ), + "leftover head.next was silently replaced", + )?; + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("head recovery evidence returned the wrong error".into()); + }; + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert!(matches!( + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(FilesystemCatalogPublicationError::HeadRecoveryRequired) + )); + drop(publisher); + assert!(store.path().join("head.next").exists()); + assert!(store.staging().join("current.seg").exists()); + assert!(!store.path().join("HEAD").exists()); + store.remove() +} From 5eec233bbb1cc824cab242aff12748fea4e2b59a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:24:57 -0700 Subject: [PATCH 13/31] Model catalog transitions and lookups --- tests/catalog_model.rs | 108 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/catalog_model.rs diff --git a/tests/catalog_model.rs b/tests/catalog_model.rs new file mode 100644 index 0000000..34724df --- /dev/null +++ b/tests/catalog_model.rs @@ -0,0 +1,108 @@ +//! Deterministic catalog transition and lookup model laws. + +mod support; + +use std::collections::BTreeMap; +use std::error::Error; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, + ChecksummedPublicationHead, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordIdentity, + SegmentRecordLimit, +}; +use support::decode_hex; + +type ReferenceCatalog = BTreeMap>; + +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const ONE_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); + +#[test] +fn generation_transitions_and_lookups_match_a_btree_map() -> Result<(), Box> { + let bundle_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let one_bytes = fixture(ONE_SEGMENT_HEX)?; + let bundle_segments = [AdmittedSegment::decode(&bundle_bytes, policy())?]; + let one_segments = [AdmittedSegment::decode(&one_bytes, policy())?]; + let empty_segments: [AdmittedSegment<'_>; 0] = []; + + let first = + CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &bundle_segments)?; + let second = CanonicalCatalog::from_segments( + CatalogGeneration::new(2)?, + Some(first.checksummed().digest()), + &one_segments, + )?; + let third = CanonicalCatalog::from_segments( + CatalogGeneration::new(3)?, + Some(second.checksummed().digest()), + &empty_segments, + )?; + + assert_snapshot_matches(&first, &bundle_segments, &model(&bundle_segments)?)?; + assert_snapshot_matches(&second, &one_segments, &model(&one_segments)?)?; + assert_snapshot_matches(&third, &empty_segments, &BTreeMap::new())?; + assert_successor(&first, &bundle_segments, &second, &one_segments, 2)?; + assert_successor(&second, &one_segments, &third, &empty_segments, 3) +} + +fn assert_snapshot_matches( + catalog: &CanonicalCatalog, + segments: &[AdmittedSegment<'_>], + expected: &ReferenceCatalog, +) -> Result<(), Box> { + let checked = catalog.checksummed(); + let admitted = checked.admit(segments)?; + let head = CanonicalPublicationHead::for_catalog(checked); + let snapshot = ChecksummedPublicationHead::decode(head.encoded())?.admit(admitted)?; + + assert_eq!(snapshot.record_count(), u64::try_from(expected.len())?); + for (identity, payload) in expected { + assert_eq!( + snapshot + .record(*identity) + .ok_or("model identity missing from snapshot")? + .payload(), + payload + ); + } + Ok(()) +} + +fn assert_successor( + current: &CanonicalCatalog, + current_segments: &[AdmittedSegment<'_>], + candidate: &CanonicalCatalog, + candidate_segments: &[AdmittedSegment<'_>], + expected_generation: u64, +) -> Result<(), Box> { + let current = current.checksummed().admit(current_segments)?; + let candidate = candidate.checksummed().admit(candidate_segments)?; + let successor = current.validate_successor(candidate)?; + assert_eq!(successor.generation().get(), expected_generation); + Ok(()) +} + +fn model(segments: &[AdmittedSegment<'_>]) -> Result> { + let mut model = BTreeMap::new(); + for segment in segments { + for record in segment.records() { + let record = record?; + if model + .insert(record.identity(), record.payload().to_vec()) + .is_some() + { + return Err("model input contains a duplicate identity".into()); + } + } + } + Ok(model) +} + +const fn policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 5ad5ca463ab4edec007c463b235d4adc4c119dac Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:30:58 -0700 Subject: [PATCH 14/31] Docs: record catalog publication evidence --- CHANGELOG.md | 23 ++++++++++-- README.md | 37 +++++++++++++------ docs/formats/segment-store-v1/README.md | 5 ++- docs/formats/segment-store-v1/publication.md | 29 +++++++++++++++ docs/formats/segment-store-v1/requirements.md | 35 ++++++++++-------- src/lib.rs | 8 ++-- 6 files changed, 100 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d3a14..3b1611a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -184,6 +184,21 @@ after its public API and format compatibility policies are established. ### Added +- Checked catalog generations; canonical catalog and publication-head codecs; + exact logical-record-to-segment admission; deterministic successor proofs; + immutable reader snapshots; and `BTreeMap` transition-model evidence for + `keep.segment-store/v1`. +- Blocking `FilesystemCatalogPublisher` publication under a persistent + kernel-managed writer lock, with pinned directory capabilities, + no-replacement immutable-pool links, complete post-link verification, + explicit file and directory synchronization, transitive `head.next` + verification, atomic `HEAD` replacement, and stale or recovery-required + refusal before mutation. +- Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact + checksummed head, catalog, and segment coordinates; refuses symbolic links, + nonregular files, malformed or conflicting bytes, dangling entries, and + resource-limit violations; and retains immutable bytes for pinned logical + reads. - Public, allocation-free `SegmentHeader` admission and emission for the exact `keep.segment-store/v1` 64-byte header, with field-complete typed refusals and golden-corpus evidence. @@ -225,10 +240,10 @@ after its public API and format compatibility policies are established. fixed-name stage into its immutable pool and durably clear the stage without promoting a publication head. Explicit discard receipts now follow synchronization of the stage's actual parent: `staging` for segment and - catalog stages, or the store root for `head.next`. Production storage - remains assigned to issues #15–#17. The golden corpus now includes a - generation-2 catalog/head pair whose predecessor field is the exact - generation-1 catalog digest. + catalog stages, or the store root for `head.next`. Segment and catalog + production are implemented; crash recovery remains assigned to issue #17. + The golden corpus now includes a generation-2 catalog/head pair whose + predecessor field is the exact generation-1 catalog digest. - A deterministic, bounded, license-safe streaming CAS benchmark corpus and release-only `cargo xtask benchmark-baseline` workflow covering all required ingestion, edit, deduplication, range-read, verification, and input diff --git a/README.md b/README.md index 643d313..ea1ae6b 100644 --- a/README.md +++ b/README.md @@ -28,21 +28,34 @@ and verifies the complete named `BlobId` before writing any bytes. Range reads load only the minimal overlapping chunks and state their narrower verification claim explicitly. -The public `keep.segment-store/v1` boundary provides exact segment, record, and -seal codecs plus explicit immutable-segment transitions. `StagedSegment` -writes only content-admitted chunk or layout records, while `AdmittedSegment` -exposes payloads only after complete framing, checksum, logical-identity, -duplicate, and physical-digest verification. `FilesystemSegmentStage` -exclusively creates the fixed `current.seg` stage without truncating existing -evidence. +The public `keep.segment-store/v1` boundary provides exact segment, record, +seal, catalog, and publication-head codecs plus explicit immutable-segment and +catalog-generation transitions. `StagedSegment` writes only content-admitted +chunk or layout records, while `AdmittedSegment` exposes payloads only after +complete framing, checksum, logical-identity, duplicate, and physical-digest +verification. `FilesystemSegmentStage` exclusively creates the fixed +`current.seg` stage without truncating existing evidence. + +`FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned +root, staging, segment-pool, and catalog-pool capabilities for the complete +blocking publication. It reopens and verifies synchronized stages, uses +no-replacement immutable-pool links, synchronizes every required file and +directory, verifies the complete `head.next` view, atomically replaces `HEAD`, +and returns a receipt only after root synchronization. +`FilesystemCatalogSnapshot` follows only the exact checksummed head, catalog, +and segment coordinates and retains caller-bounded immutable bytes for pinned +logical reads. The reference CAS is executable evidence for M2 storage laws, not a durable backend. Its committed state is process memory; process death loses it all. -The segment boundary does not publish catalogs, synchronize its containing -directory, open a durable namespace, or perform restart recovery. Catalog -publication, retention, recovery, verification of complete durable namespaces, -compaction, and garbage collection remain planned work. Presence in the -reference CAS does not claim retention, crash recovery, or durability. +The durable boundary does not yet initialize or recover a store root. +Callers must supply the exact existing `writer.lock`, `staging`, `segments`, +and `catalogs` namespace before opening a filesystem publisher. Leftover +`head.next`, staged recovery evidence, unknown namespace entries, and +ambiguous crash states remain explicit recovery work in issue #17. Retention, +complete namespace verification, compaction, and garbage collection remain +planned. Presence in the reference CAS does not claim retention, crash +recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/README.md b/docs/formats/segment-store-v1/README.md index 36aaeb2..06feafe 100644 --- a/docs/formats/segment-store-v1/README.md +++ b/docs/formats/segment-store-v1/README.md @@ -6,8 +6,9 @@ visibility, and recovery as one contract. ADR-0005 records the cross-cutting decision. These pages are a protocol commitment. Segment writing and verified reading are implemented in issue #15. -Catalog publication remains owned by issue #16, and complete executable crash -and recovery evidence remains owned by issue #17. +Catalog generation, writer-locked publication, and immutable restart snapshots +are implemented in issue #16. Store initialization and complete executable +crash and recovery evidence remain owned by issue #17. ## Core law diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index aec17e0..f39c5c0 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -116,6 +116,35 @@ renames, truncates, or replaces the lock file to break a purported stale owner. Process death releases the kernel lock. Filesystems without proven process-scoped exclusion are unsupported. +## Implemented publication boundary + +`FilesystemWriterLock::try_acquire` opens the existing regular `writer.lock` +without following symbolic links and acquires its exclusive advisory lock +without blocking. `FilesystemCatalogPublisher::open` consumes that authority +and pins the existing store root plus `staging`, `segments`, and `catalogs`. +Both operations perform blocking filesystem I/O. Neither operation initializes, +repairs, enumerates, or removes protocol state. + +`publish_catalog_generation` performs complete semantic preflight before the +first storage transition. With `FilesystemCatalogPublisher`, it then executes +the forward segment, catalog, and head protocols below. Every writable catalog +or head handle is closed before the synchronized stage is reopened read-only. +Existing immutable-pool coordinates are never replaced; their bytes are +reopened and compared against the preflighted canonical artifact before the +protocol advances. + +`FilesystemCatalogSnapshot::load` is the observational reader boundary. Its +`CatalogRestartPolicy` combines segment parser limits with a positive maximum +for aggregate retained segment bytes. The loader follows only exact +head-selected coordinates, refuses symbolic links and nonregular artifacts, +checks every length before allocation, and reconstructs logical bindings only +after all canonical bytes and physical coordinates verify. + +Issue #16 does not implement store-root initialization or explicit recovery. A +caller must supply the exact canonical directories and persistent lock file +before opening a publisher. Any retained `head.next` causes publication to +refuse before mutation and requires issue #17 recovery. + ## Forward publication protocol The writer starts with an expected current generation and catalog digest. It diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index b52a56a..f44bdd1 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -58,23 +58,24 @@ retention, or garbage collection. ## Catalog implementation evidence -Issue #16 owns catalog-generation admission, publication, and immutable reader -snapshots. The following cases are planned evidence, not current behavior. +Issue #16 implements catalog-generation admission, writer-locked filesystem +publication, and immutable reader snapshots. Store initialization and explicit +recovery remain owned by issue #17. -| ID | Planned requirement | Oracle | Planned evidence | Status | +| ID | Implemented requirement | Oracle | Executable evidence | Status | | --- | --- | --- | --- | --- | -| `KEEP-CATALOG-001` | `CatalogGeneration` admits positive values and refuses overflow when deriving a successor | Checked scalar model | `tests/catalog_generation.rs` | Planned in #16 | -| `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes | Independent golden corpus | `tests/catalog.rs`, `tests/publication_head.rs` | Planned in #16 | -| `KEEP-CATALOG-003` | Catalog entries are sorted by logical identity and duplicate keys are refused independently of input order | Ordered reference map | `tests/catalog_ordering.rs` | Planned in #16 | -| `KEEP-CATALOG-004` | Every admitted catalog location equals a verified top-level record span in the exact named segment | Segment parser and golden artifacts | `tests/catalog_locations.rs` | Planned in #16 | -| `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Planned in #16 | -| `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Planned in #16 | -| `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Planned in #16 | -| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented file and directory synchronization order | Fault-recording filesystem port | `tests/catalog_publication.rs` | Planned in #16 | -| `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Planned in #16 | -| `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Planned in #16 | +| `KEEP-CATALOG-001` | `CatalogGeneration` admits positive values and refuses overflow when deriving a successor | Checked scalar model | `tests/catalog_generation.rs` | Implemented in #16 | +| `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes | Independent golden corpus | `tests/catalog.rs`, `tests/publication_head.rs` | Implemented in #16 | +| `KEEP-CATALOG-003` | Catalog entries are sorted by logical identity and duplicate keys are refused independently of input order | Ordered reference map | `tests/catalog_ordering.rs` | Implemented in #16 | +| `KEEP-CATALOG-004` | Every admitted catalog location equals a verified top-level record span in the exact named segment | Segment parser and golden artifacts | `tests/catalog_locations.rs` | Implemented in #16 | +| `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | +| `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | +| `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | +| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented file and directory synchronization order | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | +| `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | +| `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | @@ -126,9 +127,11 @@ contains: The test-only Rust oracle reconstructs every artifact directly from these tables and formulas. The issue #15 segment implementation matches the frozen -segment corpus and adds parser fuzzing and corruption evidence. Catalog, -publication, crash-injection, recovery, and model-based generation evidence -remain owned by issues #16 and #17. +segment corpus and adds parser fuzzing and corruption evidence. Issue #16 +matches the catalog and publication-head corpus, executes the documented +publication order through a real filesystem adapter, reconstructs exact +immutable restart snapshots, and adds deterministic transition-model +evidence. Crash-injection and explicit recovery remain owned by issue #17. The format-local tradeoffs are recorded in the [colocated rationale](rationale.md). diff --git a/src/lib.rs b/src/lib.rs index a6f6152..a5e8eaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,9 +11,11 @@ //! Keep currently exposes exact logical byte and physical chunk identity, //! deterministic streaming chunk detection, canonical flat-layout identity //! and codecs, a capacity-bounded non-durable reference CAS, and explicit -//! immutable-segment writing and verified reading. Durable namespace -//! publication, retention, and recovery APIs remain intentionally absent until -//! their contracts have executable specifications. +//! immutable-segment writing and verified reading, canonical catalog +//! generations, writer-locked filesystem publication, and bounded immutable +//! restart snapshots. Store initialization, recovery, retention, and garbage +//! collection APIs remain intentionally absent until their contracts have +//! executable specifications. mod adapters; mod blob; From 2d2a7775007474e11c3eba0598527b27c6d354bd Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:42:34 -0700 Subject: [PATCH 15/31] Close segment stages before publication --- CHANGELOG.md | 3 +- README.md | 4 +- docs/formats/segment-store-v1/publication.md | 7 ++ src/adapters/catalog_publication.rs | 8 +- src/adapters/closed_segment.rs | 47 +++++++++ src/adapters/mod.rs | 4 + src/adapters/sealed_segment.rs | 18 +++- src/adapters/segment_publication.rs | 80 +++++++++++++-- src/adapters/segment_publication_error.rs | 58 +++++++++++ src/lib.rs | 11 ++- tests/catalog_filesystem_publication.rs | 29 ++++-- .../refusal_laws.rs | 23 ++--- tests/catalog_publication.rs | 12 ++- .../catalog_publication/closed_stage_laws.rs | 97 +++++++++++++++++++ tests/catalog_publication/preflight_laws.rs | 12 ++- .../catalog_publication/segment_selection.rs | 44 +++++++++ 16 files changed, 411 insertions(+), 46 deletions(-) create mode 100644 src/adapters/closed_segment.rs create mode 100644 src/adapters/segment_publication_error.rs create mode 100644 tests/catalog_publication/closed_stage_laws.rs create mode 100644 tests/catalog_publication/segment_selection.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b1611a..0524dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -193,7 +193,8 @@ after its public API and format compatibility policies are established. no-replacement immutable-pool links, complete post-link verification, explicit file and directory synchronization, transitive `head.next` verification, atomic `HEAD` replacement, and stale or recovery-required - refusal before mutation. + refusal before mutation. New segment publication consumes a handle-free + `ClosedSegment` proof before any immutable-pool link. - Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact checksummed head, catalog, and segment coordinates; refuses symbolic links, nonregular files, malformed or conflicting bytes, dangling entries, and diff --git a/README.md b/README.md index ea1ae6b..e89cd61 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ root, staging, segment-pool, and catalog-pool capabilities for the complete blocking publication. It reopens and verifies synchronized stages, uses no-replacement immutable-pool links, synchronizes every required file and directory, verifies the complete `head.next` view, atomically replaces `HEAD`, -and returns a receipt only after root synchronization. +and returns a receipt only after root synchronization. New segment publication +requires `SealedSegment::close` to consume the writable stage and bind its +handle-free `ClosedSegment` receipt to exact admitted bytes. `FilesystemCatalogSnapshot` follows only the exact checksummed head, catalog, and segment coordinates and retains caller-bounded immutable bytes for pinned logical reads. diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index f39c5c0..53047e3 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -133,6 +133,13 @@ Existing immutable-pool coordinates are never replaced; their bytes are reopened and compared against the preflighted canonical artifact before the protocol advances. +Publishing a new segment additionally requires a checked +`SegmentPublication::one` selection. The caller must first consume +`SealedSegment::close`, which drops Keep's owned writable stage before +returning a handle-free `ClosedSegment` receipt. Selection binds that receipt's +record count, byte length, and digest to the exact `AdmittedSegment` bytes. +Catalog publication cannot select an unrelated or still-open sealed stage. + `FilesystemCatalogSnapshot::load` is the observational reader boundary. Its `CatalogRestartPolicy` combines segment parser limits with a positive maximum for aggregate retained segment bytes. The loader follows only exact diff --git a/src/adapters/catalog_publication.rs b/src/adapters/catalog_publication.rs index a9276ed..08f03dc 100644 --- a/src/adapters/catalog_publication.rs +++ b/src/adapters/catalog_publication.rs @@ -26,7 +26,7 @@ pub fn publish_catalog_generation( catalog: &CanonicalCatalog, segments: &[AdmittedSegment<'_>], ) -> Result { - validate_staged_segment(segment, segments)?; + validate_staged_segment(&segment, segments)?; let checksummed = catalog.checksummed(); let admitted = checksummed.admit(segments).map_err(|source| { CatalogPublicationError::CatalogAdmission { @@ -41,7 +41,7 @@ pub fn publish_catalog_generation( .admit(admitted) .map_err(|source| CatalogPublicationError::SnapshotAdmission { source })?; catalog_publication_execution::execute_current(storage, expectation)?; - if let SegmentPublication::One(segment) = segment { + if let Some(segment) = segment.into_admitted() { catalog_publication_execution::execute_segment(storage, segment)?; } catalog_publication_execution::execute_catalog(storage, catalog, checksummed)?; @@ -98,10 +98,10 @@ const fn validate_initial( } fn validate_staged_segment( - selection: SegmentPublication<'_, '_>, + selection: &SegmentPublication<'_, '_>, segments: &[AdmittedSegment<'_>], ) -> Result<(), CatalogPublicationError> { - let SegmentPublication::One(staged) = selection else { + let Some(staged) = selection.admitted() else { return Ok(()); }; let digest = staged.digest(); diff --git a/src/adapters/closed_segment.rs b/src/adapters/closed_segment.rs new file mode 100644 index 0000000..81cb6d0 --- /dev/null +++ b/src/adapters/closed_segment.rs @@ -0,0 +1,47 @@ +//! This module owns proof that a synchronized segment stage is closed. + +use super::SegmentDigest; + +/// Metadata retained after the sealed stage's owned writable handle is closed. +/// +/// Values can be created only by consuming [`crate::SealedSegment`]. The +/// receipt carries no file handle and cannot mutate or publish the stage. +#[must_use] +#[derive(Debug)] +pub struct ClosedSegment { + record_count: u32, + segment_length: u64, + digest: SegmentDigest, +} + +impl ClosedSegment { + /// Returns the exact sealed record count. + #[must_use] + pub const fn record_count(&self) -> u32 { + self.record_count + } + + /// Returns the exact complete segment byte count. + #[must_use] + pub const fn segment_length(&self) -> u64 { + self.segment_length + } + + /// Returns the physical immutable-segment digest. + #[must_use] + pub const fn digest(&self) -> SegmentDigest { + self.digest + } + + pub(super) const fn admitted( + record_count: u32, + segment_length: u64, + digest: SegmentDigest, + ) -> Self { + Self { + record_count, + segment_length, + digest, + } + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index be33c3d..1e884eb 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -58,6 +58,7 @@ mod catalog_transition_error; mod checksummed_catalog; mod checksummed_publication_head; mod checksummed_segment_record; +mod closed_segment; mod decoded_catalog_entry; mod filesystem_catalog_artifact; mod filesystem_catalog_catalog; @@ -103,6 +104,7 @@ mod segment_header_error; mod segment_header_error_display; mod segment_identity_index; mod segment_publication; +mod segment_publication_error; mod segment_read_error; mod segment_read_error_display; mod segment_read_policy; @@ -183,6 +185,7 @@ pub use catalog_transition_error::CatalogTransitionError; pub use checksummed_catalog::ChecksummedCatalog; pub use checksummed_publication_head::ChecksummedPublicationHead; pub use checksummed_segment_record::ChecksummedSegmentRecord; +pub use closed_segment::ClosedSegment; pub use filesystem_catalog_publication_error::FilesystemCatalogPublicationError; pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; @@ -200,6 +203,7 @@ pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; pub use segment_header_error::SegmentHeaderError; pub use segment_publication::SegmentPublication; +pub use segment_publication_error::SegmentPublicationError; pub use segment_read_error::SegmentReadError; pub use segment_read_policy::SegmentReadPolicy; pub use segment_record_admission_error::SegmentRecordAdmissionError; diff --git a/src/adapters/sealed_segment.rs b/src/adapters/sealed_segment.rs index 39d5ccd..7dfdd23 100644 --- a/src/adapters/sealed_segment.rs +++ b/src/adapters/sealed_segment.rs @@ -1,6 +1,6 @@ //! Explicitly flushed and synchronized immutable segment stage. -use super::{SegmentDigest, SegmentStage}; +use super::{ClosedSegment, SegmentDigest, SegmentStage}; /// A complete segment stage whose record prefix and sealed bytes were each /// flushed and synchronized in protocol order. @@ -22,6 +22,22 @@ impl SealedSegment where S: SegmentStage, { + /// Closes the owned writable stage and returns publication-safe metadata. + /// + /// The stage has already completed both required flush-and-sync + /// transitions. This consuming operation drops the only stage value Keep + /// owns before returning a handle-free [`ClosedSegment`] receipt. + pub fn close(self) -> ClosedSegment { + let Self { + _stage: stage, + record_count, + segment_length, + digest, + } = self; + drop(stage); + ClosedSegment::admitted(record_count, segment_length, digest) + } + /// Returns the exact sealed record count. #[must_use] pub const fn record_count(&self) -> u32 { diff --git a/src/adapters/segment_publication.rs b/src/adapters/segment_publication.rs index 69afb47..86d6005 100644 --- a/src/adapters/segment_publication.rs +++ b/src/adapters/segment_publication.rs @@ -1,12 +1,76 @@ -//! Optional sealed segment transition preceding catalog publication. +//! This module owns optional closed-segment publication selection. -use super::AdmittedSegment; +use super::{AdmittedSegment, ClosedSegment, SegmentPublicationError}; /// Segment-pool work required before publishing one catalog generation. -#[derive(Clone, Copy)] -pub enum SegmentPublication<'selection, 'records> { - /// Every catalog-referenced segment is already durable in the pool. - None, - /// One fixed sealed segment stage must become a durable pool entry. - One(&'selection AdmittedSegment<'records>), +/// +/// The selected form can be created only by consuming a handle-free +/// [`ClosedSegment`] receipt and binding it to the exact admitted stage bytes. +#[must_use] +pub struct SegmentPublication<'selection, 'records> { + selected: Option>, +} + +struct SelectedSegment<'selection, 'records> { + _closed: ClosedSegment, + admitted: &'selection AdmittedSegment<'records>, +} + +impl<'selection, 'records> SegmentPublication<'selection, 'records> { + /// Selects no new segment stage because all catalog segments are durable. + pub const fn none() -> Self { + Self { selected: None } + } + + /// Binds one closed synchronized stage to its exact admitted bytes. + /// + /// # Errors + /// + /// Returns [`SegmentPublicationError`] when record count, byte length, or + /// physical digest disagrees. + pub fn one( + closed: ClosedSegment, + admitted: &'selection AdmittedSegment<'records>, + ) -> Result { + let observed_length = u64::try_from(admitted.encoded().len()).map_err(|_source| { + SegmentPublicationError::HostLength { + observed: admitted.encoded().len(), + } + })?; + if closed.record_count() != admitted.record_count() { + return Err(SegmentPublicationError::RecordCount { + expected: closed.record_count(), + observed: admitted.record_count(), + }); + } + if closed.segment_length() != observed_length { + return Err(SegmentPublicationError::SegmentLength { + expected: closed.segment_length(), + observed: observed_length, + }); + } + if closed.digest() != admitted.digest() { + return Err(SegmentPublicationError::Digest { + expected: closed.digest(), + observed: admitted.digest(), + }); + } + Ok(Self { + selected: Some(SelectedSegment { + _closed: closed, + admitted, + }), + }) + } + + pub(super) const fn admitted(&self) -> Option<&AdmittedSegment<'records>> { + match &self.selected { + Some(selected) => Some(selected.admitted), + None => None, + } + } + + pub(super) fn into_admitted(self) -> Option<&'selection AdmittedSegment<'records>> { + self.selected.map(|selected| selected.admitted) + } } diff --git a/src/adapters/segment_publication_error.rs b/src/adapters/segment_publication_error.rs new file mode 100644 index 0000000..019b065 --- /dev/null +++ b/src/adapters/segment_publication_error.rs @@ -0,0 +1,58 @@ +//! This module owns closed-stage to admitted-segment binding failures. + +use std::error::Error; +use std::fmt; + +use super::SegmentDigest; + +/// A closed stage receipt disagreed with the admitted segment selected for publication. +#[derive(Debug)] +pub enum SegmentPublicationError { + /// The admitted segment byte length cannot be represented by the protocol. + HostLength { + /// Host byte length that could not be represented. + observed: usize, + }, + /// The closed stage and admitted segment have different record counts. + RecordCount { + /// Record count proven before close. + expected: u32, + /// Record count verified from admitted bytes. + observed: u32, + }, + /// The closed stage and admitted segment have different byte lengths. + SegmentLength { + /// Byte length proven before close. + expected: u64, + /// Byte length verified from admitted bytes. + observed: u64, + }, + /// The closed stage and admitted segment have different physical digests. + Digest { + /// Digest proven before close. + expected: SegmentDigest, + /// Digest verified from admitted bytes. + observed: SegmentDigest, + }, +} + +impl fmt::Display for SegmentPublicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::HostLength { .. } => { + formatter.write_str("admitted segment length is not representable") + } + Self::RecordCount { .. } => { + formatter.write_str("closed and admitted segment record counts differ") + } + Self::SegmentLength { .. } => { + formatter.write_str("closed and admitted segment lengths differ") + } + Self::Digest { .. } => { + formatter.write_str("closed and admitted segment digests differ") + } + } + } +} + +impl Error for SegmentPublicationError {} diff --git a/src/lib.rs b/src/lib.rs index a5e8eaa..41b23fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,11 +34,12 @@ pub use adapters::{ CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, + SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index f4fffd1..2b01893 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -11,10 +11,11 @@ use std::fs; use std::path::{Path, PathBuf}; use keep::{ - AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, - FilesystemCatalogSnapshot, FilesystemWriterLock, LayoutEntryLimit, SegmentPublication, - SegmentReadPolicy, SegmentRecordLimit, publish_catalog_generation, + AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, + CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, ClosedSegment, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemSegmentStage, + FilesystemWriterLock, LayoutEntryLimit, SegmentPublication, SegmentReadPolicy, + SegmentRecordLimit, StagedSegment, publish_catalog_generation, }; use sandbox::TestDirectory; use support::decode_hex; @@ -27,21 +28,24 @@ const CATALOG_DIGEST: &str = "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03 const SEGMENT_DIGEST: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc"; const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; +type StagedFixture = (ClosedSegment, Vec); + #[test] fn successful_publication_materializes_only_the_exact_durable_view() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-success")?; - let segment_bytes = fixture(SEGMENT_HEX)?; - fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let (closed, segment_bytes) = stage_one_zero(&store)?; + assert_eq!(segment_bytes, fixture(SEGMENT_HEX)?); let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let selection = SegmentPublication::one(closed, &segments[0])?; let receipt = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::One(&segments[0]), + selection, &catalog, &segments, )?; @@ -110,6 +114,17 @@ fn restart_policy() -> Result> { )) } +fn stage_one_zero(store: &StoreFixture) -> Result> { + let stage = FilesystemSegmentStage::create(&store.staging())?; + let record = AdmittedSegmentRecord::for_chunk(&[0])?; + let closed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)? + .append(record)? + .seal()? + .close(); + let bytes = fs::read(store.staging().join("current.seg"))?; + Ok((closed, bytes)) +} + const fn maximum_segment_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index f43ae1a..a644873 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -11,27 +11,28 @@ use keep::{ }; use super::{ - EMPTY_SEGMENT_HEX, SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, + EMPTY_SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, + stage_one_zero, }; use crate::support::require_error; #[test] fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-conflict")?; - let segment_bytes = fixture(SEGMENT_HEX)?; - fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let (closed, segment_bytes) = stage_one_zero(&store)?; fs::write(store.segment_path(), fixture(EMPTY_SEGMENT_HEX)?)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let selection = SegmentPublication::one(closed, &segments[0])?; let error = require_error( publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::One(&segments[0]), + selection, &catalog, &segments, ), @@ -54,17 +55,17 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box #[test] fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-stale")?; - let segment_bytes = fixture(SEGMENT_HEX)?; - fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let (closed, segment_bytes) = stage_one_zero(&store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let selection = SegmentPublication::one(closed, &segments[0])?; let _receipt = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::One(&segments[0]), + selection, &catalog, &segments, )?; @@ -76,7 +77,7 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::None, + SegmentPublication::none(), &catalog, &segments, ), @@ -106,19 +107,19 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-next-head")?; fs::write(store.path().join("head.next"), [])?; - let segment_bytes = fixture(SEGMENT_HEX)?; - fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let (closed, segment_bytes) = stage_one_zero(&store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let selection = SegmentPublication::one(closed, &segments[0])?; let error = require_error( publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::One(&segments[0]), + selection, &catalog, &segments, ), diff --git a/tests/catalog_publication.rs b/tests/catalog_publication.rs index fb7e155..cf9dd12 100644 --- a/tests/catalog_publication.rs +++ b/tests/catalog_publication.rs @@ -1,9 +1,13 @@ //! Catalog-generation publication ordering and fault laws. +#[path = "catalog_publication/closed_stage_laws.rs"] +mod closed_stage_laws; #[path = "catalog_publication/preflight_laws.rs"] mod preflight_laws; #[path = "catalog_publication/recording_storage.rs"] pub mod recording_storage; +#[path = "catalog_publication/segment_selection.rs"] +pub mod segment_selection; mod support; use std::error::Error; @@ -24,12 +28,13 @@ fn one_generation_executes_every_durability_transition_in_order() -> Result<(), let bytes = fixture(SEGMENT_HEX)?; let fixture = publication_fixture(&bytes)?; let staged = fixture.segments.first().ok_or("missing staged segment")?; + let selection = segment_selection::for_segment(staged)?; let mut storage = RecordingStorage::succeeding(); let receipt = publish_catalog_generation( &mut storage, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::One(staged), + selection, &fixture.catalog, &fixture.segments, )?; @@ -50,12 +55,13 @@ fn every_publication_fault_stops_at_its_exact_phase() -> Result<(), Box Result<(), Box Result<(), Box> +{ + let bytes = Rc::new(RefCell::new(Vec::new())); + let dropped = Rc::new(Cell::new(false)); + let stage = ObservableStage::new(Rc::clone(&bytes), Rc::clone(&dropped)); + let record = AdmittedSegmentRecord::for_chunk(&[0])?; + let sealed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)? + .append(record)? + .seal()?; + + assert!(!dropped.get()); + let closed = sealed.close(); + assert!(dropped.get()); + let encoded = bytes.borrow(); + let admitted = AdmittedSegment::decode(&encoded, policy())?; + let _selection = SegmentPublication::one(closed, &admitted)?; + Ok(()) +} + +#[test] +fn closed_stage_metadata_must_match_the_admitted_bytes() -> Result<(), Box> { + let bytes = Rc::new(RefCell::new(Vec::new())); + let dropped = Rc::new(Cell::new(false)); + let stage = ObservableStage::new(bytes, dropped); + let closed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)? + .seal()? + .close(); + let admitted_bytes = fixture(SEGMENT_HEX)?; + let admitted = AdmittedSegment::decode(&admitted_bytes, policy())?; + let error = require_error( + SegmentPublication::one(closed, &admitted), + "mismatched closed stage was selected for publication", + )?; + + assert!(matches!( + error, + SegmentPublicationError::RecordCount { + expected: 0, + observed: 1, + } + )); + Ok(()) +} + +struct ObservableStage { + bytes: Rc>>, + dropped: Rc>, +} + +impl ObservableStage { + const fn new(bytes: Rc>>, dropped: Rc>) -> Self { + Self { bytes, dropped } + } +} + +impl Write for ObservableStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for ObservableStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Drop for ObservableStage { + fn drop(&mut self) { + self.dropped.set(true); + } +} + +const fn policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/catalog_publication/preflight_laws.rs b/tests/catalog_publication/preflight_laws.rs index 2a72d41..a83aa1d 100644 --- a/tests/catalog_publication/preflight_laws.rs +++ b/tests/catalog_publication/preflight_laws.rs @@ -9,6 +9,7 @@ use keep::{ }; use super::recording_storage::RecordingStorage; +use super::segment_selection; use super::{EMPTY_SEGMENT_HEX, SEGMENT_HEX, fixture, maximum_policy, publication_fixture}; use crate::support::require_error; @@ -19,12 +20,13 @@ fn staged_segment_must_belong_to_the_admitted_set_before_io() -> Result<(), Box< let staged_bytes = fixture(EMPTY_SEGMENT_HEX)?; let staged = AdmittedSegment::decode(&staged_bytes, maximum_policy())?; let expected = staged.digest(); + let selection = segment_selection::for_segment(&staged)?; let mut storage = RecordingStorage::succeeding(); let error = require_error( publish_catalog_generation( &mut storage, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::One(&staged), + selection, &publication.catalog, &publication.segments, ), @@ -49,7 +51,7 @@ fn catalog_location_refusal_precedes_every_storage_call() -> Result<(), Box Result<(), publish_catalog_generation( &mut storage, expectation, - SegmentPublication::None, + SegmentPublication::none(), &publication.catalog, &publication.segments, ), @@ -106,7 +108,7 @@ fn current_snapshot_requires_and_admits_only_its_exact_successor() -> Result<(), publish_catalog_generation( &mut storage, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::None, + SegmentPublication::none(), &successor, &publication.segments, ), @@ -121,7 +123,7 @@ fn current_snapshot_requires_and_admits_only_its_exact_successor() -> Result<(), let receipt = publish_catalog_generation( &mut storage, expectation, - SegmentPublication::None, + SegmentPublication::none(), &successor, &publication.segments, )?; diff --git a/tests/catalog_publication/segment_selection.rs b/tests/catalog_publication/segment_selection.rs new file mode 100644 index 0000000..1773675 --- /dev/null +++ b/tests/catalog_publication/segment_selection.rs @@ -0,0 +1,44 @@ +//! Checked closed-stage publication selections for catalog tests. + +use std::error::Error; +use std::io::{self, Write}; + +use keep::{AdmittedSegment, SegmentPublication, SegmentRecordLimit, SegmentStage, StagedSegment}; + +/// Reconstructs and closes the exact admitted segment for publication tests. +/// +/// # Errors +/// +/// Returns the exact record iteration, staging, sealing, or receipt-binding +/// failure. +pub fn for_segment<'selection, 'records>( + segment: &'selection AdmittedSegment<'records>, +) -> Result, Box> { + let mut staged = StagedSegment::begin(MemoryStage::default(), SegmentRecordLimit::MAXIMUM)?; + for record in segment.records() { + staged = staged.append(record?)?; + } + SegmentPublication::one(staged.seal()?.close(), segment).map_err(Into::into) +} + +#[derive(Default)] +struct MemoryStage { + bytes: Vec, +} + +impl Write for MemoryStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} From 039ce7ee6118476e20cab6e867b6eab8219e7e77 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:50:41 -0700 Subject: [PATCH 16/31] Bind segment staging to writer authority --- CHANGELOG.md | 8 ++- README.md | 6 +- docs/formats/segment-store-v1/publication.md | 7 +- src/adapters/filesystem_catalog_publisher.rs | 20 +++++- src/adapters/filesystem_segment_stage.rs | 55 +++++++------- tests/catalog_filesystem_publication.rs | 19 ++--- .../refusal_laws.rs | 18 ++--- tests/segment_filesystem_stage.rs | 71 +++++++++---------- 8 files changed, 112 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0524dc7..fb400b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -220,9 +220,11 @@ after its public API and format compatibility policies are established. for exact append-only record writing, streaming seal construction, explicit prefix/sealed flush-and-sync order, phase-typed I/O refusals, and a fallibly reserved membership index for sublinear duplicate admission. -- Exclusive `FilesystemSegmentStage` creation for the fixed `current.seg` - staging name, with atomic no-replacement admission, preserved existing - evidence, zero-origin writing, and no implicit cleanup from `Drop`. +- Writer-authorized `FilesystemSegmentStage` creation for the fixed + `current.seg` staging name, with a lifetime that retains the + `FilesystemCatalogPublisher` lock, atomic no-replacement admission, + preserved existing evidence, zero-origin writing, and no implicit cleanup + from `Drop`. - Rust cargo-fuzz coverage for the public segment header, record header, complete record, seal, and complete-segment parser boundaries, seeded from the canonical version-1 segment fixtures through `cargo xtask`. diff --git a/README.md b/README.md index e89cd61..d262e87 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,10 @@ seal, catalog, and publication-head codecs plus explicit immutable-segment and catalog-generation transitions. `StagedSegment` writes only content-admitted chunk or layout records, while `AdmittedSegment` exposes payloads only after complete framing, checksum, logical-identity, duplicate, and physical-digest -verification. `FilesystemSegmentStage` exclusively creates the fixed -`current.seg` stage without truncating existing evidence. +verification. A locked `FilesystemCatalogPublisher` exclusively creates the +fixed `current.seg` stage without truncating existing evidence, and the +`FilesystemSegmentStage` lifetime keeps that writer authority borrowed until +the writable stage closes. `FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned root, staging, segment-pool, and catalog-pool capabilities for the complete diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index 53047e3..550f9b1 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -139,6 +139,9 @@ Publishing a new segment additionally requires a checked returning a handle-free `ClosedSegment` receipt. Selection binds that receipt's record count, byte length, and digest to the exact `AdmittedSegment` bytes. Catalog publication cannot select an unrelated or still-open sealed stage. +`FilesystemCatalogPublisher::create_segment_stage` is the only public +filesystem-stage constructor. Its returned lifetime keeps the acquired writer +authority borrowed while `current.seg` remains writable. `FilesystemCatalogSnapshot::load` is the observational reader boundary. Its `CatalogRestartPolicy` combines segment parser limits with a positive maximum @@ -174,8 +177,8 @@ preceding namespace mutation became durable merely because it was issued. ### Seal each new segment -1. Create `staging/current.seg` exclusively - (`KEEP-CRASH-001`). +1. Under the acquired writer authority, create `staging/current.seg` + exclusively (`KEEP-CRASH-001`). 2. Write the complete 64-byte header (`KEEP-CRASH-002`). 3. Append each complete record and checksum (`KEEP-CRASH-003`, with an occurrence counter for tests). diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 58fe8ba..ec0f2d4 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -5,7 +5,9 @@ use std::io; use cap_fs_ext::DirExt; use cap_std::fs::{Dir, File}; -use super::{CatalogRestartPolicy, FilesystemWriterLock}; +use super::{ + CatalogRestartPolicy, FilesystemSegmentStage, FilesystemWriterLock, SegmentStageCreateError, +}; pub(super) const CURRENT_SEGMENT: &str = "current.seg"; pub(super) const CURRENT_CATALOG: &str = "current.cat"; @@ -53,4 +55,20 @@ impl FilesystemCatalogPublisher { head_stage: None, }) } + + /// Exclusively creates `staging/current.seg` under this writer authority. + /// + /// The returned stage borrows this publisher until the stage is dropped or + /// consumed by [`crate::StagedSegment`] and explicitly closed after + /// sealing. Creation performs blocking, capability-relative filesystem I/O. + /// + /// # Errors + /// + /// Returns [`SegmentStageCreateError`] without opening or truncating an + /// existing filesystem entry. + pub fn create_segment_stage( + &self, + ) -> Result, SegmentStageCreateError> { + FilesystemSegmentStage::create(self) + } } diff --git a/src/adapters/filesystem_segment_stage.rs b/src/adapters/filesystem_segment_stage.rs index 288eac7..bdb9159 100644 --- a/src/adapters/filesystem_segment_stage.rs +++ b/src/adapters/filesystem_segment_stage.rs @@ -1,49 +1,44 @@ -//! Exclusive fixed-name filesystem segment stage. +//! Writer-authorized fixed-name filesystem segment stage. -use std::fs::{File, OpenOptions}; use std::io::{self, Write}; -use std::path::Path; -use super::{SegmentStage, SegmentStageCreateError}; +use cap_std::fs::File; -const STAGE_NAME: &str = "current.seg"; +use super::filesystem_catalog_artifact; +use super::filesystem_catalog_publisher::CURRENT_SEGMENT; +use super::{FilesystemCatalogPublisher, SegmentStage, SegmentStageCreateError}; -/// An exclusively created empty `current.seg` staging file. +/// An exclusively created empty `current.seg` staging file under writer authority. /// /// Creation uses one atomic no-replacement filesystem operation. A successful /// value therefore owns a new regular file positioned at byte zero. Dropping /// the value closes the file but deliberately leaves its bytes and name for /// explicit recovery. /// -/// This type does not create the staging directory, synchronize its directory -/// entry, publish the file, or establish that the surrounding filesystem -/// satisfies Keep's complete platform contract. -pub struct FilesystemSegmentStage { +/// The lifetime keeps the locked publisher borrowed until the writable file is +/// closed. This type does not synchronize the staging-directory entry, publish +/// the file, or establish that the surrounding filesystem satisfies Keep's +/// complete platform contract. +pub struct FilesystemSegmentStage<'publisher> { file: File, + _publisher: &'publisher FilesystemCatalogPublisher, } -impl FilesystemSegmentStage { - /// Exclusively creates `current.seg` beneath `staging_directory`. - /// - /// This operation performs blocking filesystem I/O and allocates the - /// platform path buffer required to append the fixed stage name. - /// - /// # Errors - /// - /// Returns [`SegmentStageCreateError`] without opening or truncating an - /// existing filesystem entry. - pub fn create(staging_directory: &Path) -> Result { - let path = staging_directory.join(STAGE_NAME); - let file = OpenOptions::new() - .write(true) - .create_new(true) - .open(path) - .map_err(|source| SegmentStageCreateError::Create { source })?; - Ok(Self { file }) +impl<'publisher> FilesystemSegmentStage<'publisher> { + pub(super) fn create( + publisher: &'publisher FilesystemCatalogPublisher, + ) -> Result { + let file = + filesystem_catalog_artifact::create_exclusive(&publisher.staging, CURRENT_SEGMENT) + .map_err(|source| SegmentStageCreateError::Create { source })?; + Ok(Self { + file, + _publisher: publisher, + }) } } -impl Write for FilesystemSegmentStage { +impl Write for FilesystemSegmentStage<'_> { fn write(&mut self, bytes: &[u8]) -> io::Result { self.file.write(bytes) } @@ -53,7 +48,7 @@ impl Write for FilesystemSegmentStage { } } -impl SegmentStage for FilesystemSegmentStage { +impl SegmentStage for FilesystemSegmentStage<'_> { fn synchronize(&mut self) -> io::Result<()> { self.file.sync_all() } diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index 2b01893..f731207 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -13,9 +13,9 @@ use std::path::{Path, PathBuf}; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, ClosedSegment, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemSegmentStage, - FilesystemWriterLock, LayoutEntryLimit, SegmentPublication, SegmentReadPolicy, - SegmentRecordLimit, StagedSegment, publish_catalog_generation, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemWriterLock, LayoutEntryLimit, + SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, + publish_catalog_generation, }; use sandbox::TestDirectory; use support::decode_hex; @@ -33,13 +33,13 @@ type StagedFixture = (ClosedSegment, Vec); #[test] fn successful_publication_materializes_only_the_exact_durable_view() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-success")?; - let (closed, segment_bytes) = stage_one_zero(&store)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; assert_eq!(segment_bytes, fixture(SEGMENT_HEX)?); let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; let selection = SegmentPublication::one(closed, &segments[0])?; let receipt = publish_catalog_generation( @@ -114,8 +114,11 @@ fn restart_policy() -> Result> { )) } -fn stage_one_zero(store: &StoreFixture) -> Result> { - let stage = FilesystemSegmentStage::create(&store.staging())?; +fn stage_one_zero( + publisher: &FilesystemCatalogPublisher, + store: &StoreFixture, +) -> Result> { + let stage = publisher.create_segment_stage()?; let record = AdmittedSegmentRecord::for_chunk(&[0])?; let closed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)? .append(record)? diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index a644873..fe437b1 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -19,13 +19,13 @@ use crate::support::require_error; #[test] fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-conflict")?; - let (closed, segment_bytes) = stage_one_zero(&store)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; fs::write(store.segment_path(), fixture(EMPTY_SEGMENT_HEX)?)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; let selection = SegmentPublication::one(closed, &segments[0])?; let error = require_error( @@ -55,12 +55,12 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box #[test] fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-stale")?; - let (closed, segment_bytes) = stage_one_zero(&store)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; let selection = SegmentPublication::one(closed, &segments[0])?; let _receipt = publish_catalog_generation( &mut publisher, @@ -107,12 +107,12 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-next-head")?; fs::write(store.path().join("head.next"), [])?; - let (closed, segment_bytes) = stage_one_zero(&store)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; let selection = SegmentPublication::one(closed, &segments[0])?; let error = require_error( diff --git a/tests/segment_filesystem_stage.rs b/tests/segment_filesystem_stage.rs index dd51b16..470bd8f 100644 --- a/tests/segment_filesystem_stage.rs +++ b/tests/segment_filesystem_stage.rs @@ -7,12 +7,11 @@ mod support; use std::error::Error; use std::fs; use std::io::ErrorKind; -use std::sync::{Arc, Barrier}; -use std::thread; use keep::{ - AdmittedSegmentRecord, FilesystemSegmentStage, SegmentHeader, SegmentRecordLimit, - SegmentStageCreateError, StagedSegment, + AdmittedSegmentRecord, CatalogRestartByteLimit, CatalogRestartPolicy, + FilesystemCatalogPublisher, FilesystemWriterLock, LayoutEntryLimit, SegmentHeader, + SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, StagedSegment, }; use sandbox::TestDirectory; use support::decode_hex; @@ -24,12 +23,12 @@ const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/em #[test] fn exclusive_creation_never_truncates_existing_stage() -> Result<(), Box> { let sandbox = TestDirectory::create("exclusive-create-refusal")?; + let publisher = open_publisher(&sandbox)?; let staging = sandbox.path().join("staging"); - fs::create_dir(&staging)?; let stage_path = staging.join("current.seg"); fs::write(&stage_path, b"preserved evidence")?; - let error = match FilesystemSegmentStage::create(&staging) { + let error = match publisher.create_segment_stage() { Ok(_stage) => return Err("existing stage was replaced".into()), Err(error) => error, }; @@ -39,39 +38,20 @@ fn exclusive_creation_never_truncates_existing_stage() -> Result<(), Box Result<(), Box> { - let sandbox = TestDirectory::create("exclusive-create-race")?; +fn repeated_stage_creation_admits_exactly_one_owner() -> Result<(), Box> { + let sandbox = TestDirectory::create("exclusive-create-repeat")?; + let publisher = open_publisher(&sandbox)?; let staging = sandbox.path().join("staging"); - fs::create_dir(&staging)?; - let barrier = Arc::new(Barrier::new(3)); - let contender = |barrier: Arc| { - let staging = staging.clone(); - thread::spawn(move || { - barrier.wait(); - FilesystemSegmentStage::create(&staging) - }) - }; - let first = contender(Arc::clone(&barrier)); - let second = contender(Arc::clone(&barrier)); - barrier.wait(); - let first = first.join().map_err(|_panic| "first contender panicked")?; - let second = second - .join() - .map_err(|_panic| "second contender panicked")?; - let refusal = match (first, second) { - (Ok(stage), Err(error)) | (Err(error), Ok(stage)) => { - drop(stage); - error - } - (Ok(_first), Ok(_second)) => return Err("both stage contenders were admitted".into()), - (Err(first), Err(second)) => { - return Err(format!("both stage contenders were refused: {first}; {second}").into()); - } + let first = publisher.create_segment_stage()?; + let refusal = match publisher.create_segment_stage() { + Ok(_second) => return Err("both stage contenders were admitted".into()), + Err(error) => error, }; assert!(matches!( @@ -80,6 +60,8 @@ fn racing_stage_creation_admits_exactly_one_owner() -> Result<(), Box if source.kind() == ErrorKind::AlreadyExists )); assert!(staging.join("current.seg").is_file()); + drop(first); + drop(publisher); sandbox.remove()?; Ok(()) } @@ -87,9 +69,9 @@ fn racing_stage_creation_admits_exactly_one_owner() -> Result<(), Box #[test] fn exclusive_stage_starts_at_zero_and_retains_exact_sealed_bytes() -> Result<(), Box> { let sandbox = TestDirectory::create("exclusive-create-success")?; + let publisher = open_publisher(&sandbox)?; let staging = sandbox.path().join("staging"); - fs::create_dir(&staging)?; - let stage = FilesystemSegmentStage::create(&staging)?; + let stage = publisher.create_segment_stage()?; let staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; let staged = staged.append(AdmittedSegmentRecord::for_chunk(&[0])?)?; let sealed = staged.seal()?; @@ -101,6 +83,7 @@ fn exclusive_stage_starts_at_zero_and_retains_exact_sealed_bytes() -> Result<(), )?; assert_eq!(fs::read(staging.join("current.seg"))?, canonical); + drop(publisher); sandbox.remove()?; Ok(()) } @@ -108,9 +91,9 @@ fn exclusive_stage_starts_at_zero_and_retains_exact_sealed_bytes() -> Result<(), #[test] fn dropping_an_unsealed_stage_preserves_the_reusable_prefix() -> Result<(), Box> { let sandbox = TestDirectory::create("unsealed-prefix-preservation")?; + let publisher = open_publisher(&sandbox)?; let staging = sandbox.path().join("staging"); - fs::create_dir(&staging)?; - let stage = FilesystemSegmentStage::create(&staging)?; + let stage = publisher.create_segment_stage()?; let staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; drop(staged); let empty_segment = decode_hex( @@ -123,6 +106,20 @@ fn dropping_an_unsealed_stage_preserves_the_reusable_prefix() -> Result<(), Box< .ok_or("empty segment fixture lacks its header")?; assert_eq!(fs::read(staging.join("current.seg"))?, header); + drop(publisher); sandbox.remove()?; Ok(()) } + +fn open_publisher(sandbox: &TestDirectory) -> Result> { + fs::write(sandbox.path().join("writer.lock"), [])?; + fs::create_dir(sandbox.path().join("staging"))?; + fs::create_dir(sandbox.path().join("segments"))?; + fs::create_dir(sandbox.path().join("catalogs"))?; + let lock = FilesystemWriterLock::try_acquire(sandbox.path())?; + let policy = CatalogRestartPolicy::new( + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), + CatalogRestartByteLimit::new(1_048_576)?, + ); + Ok(FilesystemCatalogPublisher::open(lock, policy)?) +} From 8ed7533f3677ff2b24873f0fda4bd48c659b22b2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 01:54:23 -0700 Subject: [PATCH 17/31] Retain writer authority through publisher teardown --- CHANGELOG.md | 3 ++- docs/formats/segment-store-v1/publication.md | 3 ++- src/adapters/filesystem_catalog_publisher.rs | 10 ++++++---- tests/catalog_filesystem_publication.rs | 19 +++++++++++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb400b8..5b657c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -194,7 +194,8 @@ after its public API and format compatibility policies are established. explicit file and directory synchronization, transitive `head.next` verification, atomic `HEAD` replacement, and stale or recovery-required refusal before mutation. New segment publication consumes a handle-free - `ClosedSegment` proof before any immutable-pool link. + `ClosedSegment` proof before any immutable-pool link, and publisher teardown + closes every retained writable handle before releasing writer authority. - Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact checksummed head, catalog, and segment coordinates; refuses symbolic links, nonregular files, malformed or conflicting bytes, dangling entries, and diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index 550f9b1..0592360 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -131,7 +131,8 @@ the forward segment, catalog, and head protocols below. Every writable catalog or head handle is closed before the synchronized stage is reopened read-only. Existing immutable-pool coordinates are never replaced; their bytes are reopened and compared against the preflighted canonical artifact before the -protocol advances. +protocol advances. Publisher teardown closes retained writable handles and +pinned directory capabilities before releasing the writer lock. Publishing a new segment additionally requires a checked `SegmentPublication::one` selection. The caller must first consume diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index ec0f2d4..071fb48 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -18,11 +18,10 @@ pub(super) const NEXT_HEAD: &str = "head.next"; /// /// The publisher owns the writer lock and pinned root, staging, segment-pool, /// and catalog-pool directory capabilities until it is dropped. Dropping it -/// closes open stages and releases the writer lock but never publishes, -/// removes, truncates, or repairs protocol state. +/// closes open stages and directory capabilities before releasing the writer +/// lock, but never publishes, removes, truncates, or repairs protocol state. #[must_use] pub struct FilesystemCatalogPublisher { - pub(super) _lock: FilesystemWriterLock, pub(super) root: Dir, pub(super) staging: Dir, pub(super) segments: Dir, @@ -30,6 +29,9 @@ pub struct FilesystemCatalogPublisher { pub(super) policy: CatalogRestartPolicy, pub(super) catalog_stage: Option, pub(super) head_stage: Option, + // Fields drop in declaration order. Writer authority must outlive every + // directory capability and retained writable stage. + pub(super) _lock: FilesystemWriterLock, } impl FilesystemCatalogPublisher { @@ -45,7 +47,6 @@ impl FilesystemCatalogPublisher { let segments = root.open_dir_nofollow("segments")?; let catalogs = root.open_dir_nofollow("catalogs")?; Ok(Self { - _lock: lock, root, staging, segments, @@ -53,6 +54,7 @@ impl FilesystemCatalogPublisher { policy, catalog_stage: None, head_stage: None, + _lock: lock, }) } diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index f731207..75883d7 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -135,3 +135,22 @@ const fn maximum_segment_policy() -> SegmentReadPolicy { fn fixture(hex: &str) -> Result, Box> { decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) } + +#[test] +fn publisher_drop_closes_writable_stages_before_releasing_writer_authority() +-> Result<(), Box> { + let source = include_str!("../src/adapters/filesystem_catalog_publisher.rs"); + let catalog_stage = source + .find("pub(super) catalog_stage:") + .ok_or("publisher must retain the catalog stage")?; + let head_stage = source + .find("pub(super) head_stage:") + .ok_or("publisher must retain the head stage")?; + let writer_lock = source + .find("pub(super) _lock:") + .ok_or("publisher must retain writer authority")?; + + assert!(catalog_stage < writer_lock); + assert!(head_stage < writer_lock); + Ok(()) +} From aad03099c043a993024c85db631d1d4f3013bbcb Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:04:10 -0700 Subject: [PATCH 18/31] Bound catalog admission record scans --- CHANGELOG.md | 3 +- docs/formats/segment-store-v1/catalog.md | 7 +- src/adapters/catalog_admission.rs | 75 +++++------ src/adapters/catalog_admission_error.rs | 2 +- src/adapters/catalog_allocation_phase.rs | 3 + src/adapters/catalog_entry_plan.rs | 127 ++++++++++++++++++ src/adapters/mod.rs | 1 + tests/catalog_locations.rs | 13 ++ .../tests/segment_store_protocol_contract.rs | 3 +- 9 files changed, 184 insertions(+), 50 deletions(-) create mode 100644 src/adapters/catalog_entry_plan.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b657c1..1b9a044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -185,7 +185,8 @@ after its public API and format compatibility policies are established. ### Added - Checked catalog generations; canonical catalog and publication-head codecs; - exact logical-record-to-segment admission; deterministic successor proofs; + exact logical-record-to-segment admission with one bounded physical lookup + plan and one scan per referenced segment; deterministic successor proofs; immutable reader snapshots; and `BTreeMap` transition-model evidence for `keep.segment-store/v1`. - Blocking `FilesystemCatalogPublisher` publication under a persistent diff --git a/docs/formats/segment-store-v1/catalog.md b/docs/formats/segment-store-v1/catalog.md index b723f40..2e16e25 100644 --- a/docs/formats/segment-store-v1/catalog.md +++ b/docs/formats/segment-store-v1/catalog.md @@ -78,9 +78,10 @@ The named segment must exist at the digest-derived immutable-pool name and verify completely. The record at the declared span must reproduce the entry's kind, identity, lengths, and checksum exactly. -Catalog admission scans the complete segment grammar from byte 64 through the -declared record count and records every top-level record span before admitting -locations. Each `(record_offset, record_length)` pair must equal one discovered +Catalog admission creates one bounded plan over the declared entries, groups +that plan by segment digest and physical offset, and scans each referenced +segment exactly once. Each scan runs from byte 64 through the declared record +count. Each `(record_offset, record_length)` pair must equal one discovered top-level record span. A location into a record header, payload, checksum, or segment seal is refused even when those embedded bytes independently resemble a valid record. diff --git a/src/adapters/catalog_admission.rs b/src/adapters/catalog_admission.rs index 4fad970..a5bafd1 100644 --- a/src/adapters/catalog_admission.rs +++ b/src/adapters/catalog_admission.rs @@ -1,5 +1,6 @@ //! Binding of catalog-local coordinates to exact admitted segment records. +use super::catalog_entry_plan::{self, CatalogEntryPlan}; use super::{ AdmittedCatalog, AdmittedSegment, AdmittedSegmentRecord, CatalogAdmissionError, CatalogAllocationPhase, CatalogRecordBinding, ChecksummedCatalog, DecodedCatalogEntry, @@ -22,6 +23,9 @@ pub(super) fn admit<'catalog, 'records>( }); } let segment_index = index_segments(segments)?; + let mut plan = plan_entries(catalog, &segment_index, requested)?; + catalog_entry_plan::bind(&mut plan)?; + plan.sort_unstable_by_key(CatalogEntryPlan::ordinal); let mut bindings = Vec::new(); bindings .try_reserve_exact(requested) @@ -30,17 +34,35 @@ pub(super) fn admit<'catalog, 'records>( requested, source, })?; + for planned in plan { + let (entry, record) = planned.into_bound()?; + validate_record(entry, record)?; + bindings.push(CatalogRecordBinding::new(entry.identity(), record)); + } + Ok(AdmittedCatalog::from_verified_parts(catalog, bindings)) +} + +fn plan_entries<'segments, 'records>( + catalog: ChecksummedCatalog<'_>, + segments: &[&'segments AdmittedSegment<'records>], + requested: usize, +) -> Result>, CatalogAdmissionError> { + let mut plan = Vec::new(); + plan.try_reserve_exact(requested) + .map_err(|source| CatalogAdmissionError::Allocation { + phase: CatalogAllocationPhase::EntryPlan, + requested, + source, + })?; let entries = catalog .entries() .map_err(|source| CatalogAdmissionError::Catalog { source })?; - for entry in entries { + for (ordinal, entry) in entries.enumerate() { let entry = entry.map_err(|source| CatalogAdmissionError::Catalog { source })?; - let segment = find_segment(&segment_index, entry.segment_digest())?; - let record = locate_record(segment, entry)?; - validate_record(entry, record)?; - bindings.push(CatalogRecordBinding::new(entry.identity(), record)); + let segment = find_segment(segments, entry.segment_digest())?; + plan.push(CatalogEntryPlan::new(ordinal, entry, segment)); } - Ok(AdmittedCatalog::from_verified_parts(catalog, bindings)) + Ok(plan) } fn index_segments<'slice, 'records>( @@ -70,10 +92,10 @@ fn index_segments<'slice, 'records>( Ok(indexed) } -fn find_segment<'slice, 'records>( - segments: &'slice [&AdmittedSegment<'records>], +fn find_segment<'segments, 'records>( + segments: &[&'segments AdmittedSegment<'records>], digest: SegmentDigest, -) -> Result<&'slice AdmittedSegment<'records>, CatalogAdmissionError> { +) -> Result<&'segments AdmittedSegment<'records>, CatalogAdmissionError> { let index = segments .binary_search_by_key(&digest, |segment| segment.digest()) .map_err(|_source| CatalogAdmissionError::MissingSegment { digest })?; @@ -83,41 +105,6 @@ fn find_segment<'slice, 'records>( .ok_or(CatalogAdmissionError::MissingSegment { digest }) } -fn locate_record<'records>( - segment: &AdmittedSegment<'records>, - entry: DecodedCatalogEntry, -) -> Result, CatalogAdmissionError> { - let digest = segment.digest(); - let mut cursor = segment.record_cursor(); - let mut found = None; - while let Some(located) = - cursor - .next_record() - .map_err(|source| CatalogAdmissionError::Segment { - digest, - source: Box::new(source), - })? - { - if located.offset == entry.record_offset() - && located.record.header().record_length() == entry.record_length() - { - found = Some(located.record); - } - } - cursor - .finish() - .map_err(|source| CatalogAdmissionError::Segment { - digest, - source: Box::new(source), - })?; - found.ok_or_else(|| CatalogAdmissionError::LocationNotTopLevel { - identity: entry.identity(), - segment_digest: entry.segment_digest(), - record_offset: entry.record_offset(), - record_length: entry.record_length().get(), - }) -} - fn validate_record( entry: DecodedCatalogEntry, record: AdmittedSegmentRecord<'_>, diff --git a/src/adapters/catalog_admission_error.rs b/src/adapters/catalog_admission_error.rs index 52cbd42..bc8eee2 100644 --- a/src/adapters/catalog_admission_error.rs +++ b/src/adapters/catalog_admission_error.rs @@ -27,7 +27,7 @@ pub enum CatalogAdmissionError { /// Caller-supplied segment count. observed: usize, }, - /// A bounded segment-index or record-binding allocation failed. + /// A bounded segment-index, entry-plan, or record-binding allocation failed. Allocation { /// Semantic allocation phase. phase: CatalogAllocationPhase, diff --git a/src/adapters/catalog_allocation_phase.rs b/src/adapters/catalog_allocation_phase.rs index a8b79e3..3c781d7 100644 --- a/src/adapters/catalog_allocation_phase.rs +++ b/src/adapters/catalog_allocation_phase.rs @@ -7,6 +7,8 @@ use std::fmt; pub enum CatalogAllocationPhase { /// Sorted borrowed index over caller-supplied admitted segments. SegmentIndex, + /// Bounded physical lookup plan over canonical catalog entries. + EntryPlan, /// Logical-identity bindings retained by the admitted catalog. RecordBindings, } @@ -15,6 +17,7 @@ impl fmt::Display for CatalogAllocationPhase { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::SegmentIndex => formatter.write_str("segment index"), + Self::EntryPlan => formatter.write_str("entry plan"), Self::RecordBindings => formatter.write_str("record bindings"), } } diff --git a/src/adapters/catalog_entry_plan.rs b/src/adapters/catalog_entry_plan.rs new file mode 100644 index 0000000..3520836 --- /dev/null +++ b/src/adapters/catalog_entry_plan.rs @@ -0,0 +1,127 @@ +//! This module owns one-pass physical record lookup for catalog admission. + +use super::{ + AdmittedSegment, AdmittedSegmentRecord, CatalogAdmissionError, DecodedCatalogEntry, + SegmentDigest, +}; + +pub(super) struct CatalogEntryPlan<'segments, 'records> { + ordinal: usize, + entry: DecodedCatalogEntry, + segment: &'segments AdmittedSegment<'records>, + record: Option>, +} + +impl<'segments, 'records> CatalogEntryPlan<'segments, 'records> { + pub(super) const fn new( + ordinal: usize, + entry: DecodedCatalogEntry, + segment: &'segments AdmittedSegment<'records>, + ) -> Self { + Self { + ordinal, + entry, + segment, + record: None, + } + } + + pub(super) const fn ordinal(&self) -> usize { + self.ordinal + } + + pub(super) const fn physical_order(&self) -> (SegmentDigest, u64, usize) { + ( + self.entry.segment_digest(), + self.entry.record_offset(), + self.ordinal, + ) + } + + pub(super) fn into_bound( + self, + ) -> Result<(DecodedCatalogEntry, AdmittedSegmentRecord<'records>), CatalogAdmissionError> { + let record = self + .record + .ok_or_else(|| CatalogAdmissionError::LocationNotTopLevel { + identity: self.entry.identity(), + segment_digest: self.entry.segment_digest(), + record_offset: self.entry.record_offset(), + record_length: self.entry.record_length().get(), + })?; + Ok((self.entry, record)) + } + + const fn segment_digest(&self) -> SegmentDigest { + self.entry.segment_digest() + } + + const fn record_offset(&self) -> u64 { + self.entry.record_offset() + } + + fn bind_if_length_matches(&mut self, offset: u64, record: AdmittedSegmentRecord<'records>) { + let length_matches = record.header().record_length() == self.entry.record_length(); + if self.entry.record_offset() == offset && length_matches { + self.record = Some(record); + } + } +} + +pub(super) fn bind(entries: &mut [CatalogEntryPlan<'_, '_>]) -> Result<(), CatalogAdmissionError> { + entries.sort_unstable_by_key(CatalogEntryPlan::physical_order); + for group in + entries.chunk_by_mut(|first, second| first.segment_digest() == second.segment_digest()) + { + bind_segment(group)?; + } + Ok(()) +} + +fn bind_segment(entries: &mut [CatalogEntryPlan<'_, '_>]) -> Result<(), CatalogAdmissionError> { + let Some(first) = entries.first() else { + return Ok(()); + }; + let segment = first.segment; + let digest = segment.digest(); + let mut pending = entries.iter_mut().peekable(); + let mut cursor = segment.record_cursor(); + while let Some(located) = + cursor + .next_record() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })? + { + skip_preceding_entries(&mut pending, located.offset); + bind_entries_at_offset(&mut pending, located.offset, located.record); + } + cursor + .finish() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + }) +} + +fn skip_preceding_entries( + entries: &mut std::iter::Peekable>>, + record_offset: u64, +) { + while matches!(entries.peek(), Some(entry) if entry.record_offset() < record_offset) { + let _skipped = entries.next(); + } +} + +fn bind_entries_at_offset<'records>( + entries: &mut std::iter::Peekable>>, + record_offset: u64, + record: AdmittedSegmentRecord<'records>, +) { + while matches!(entries.peek(), Some(entry) if entry.record_offset() == record_offset) { + if let Some(entry) = entries.next() { + entry.bind_if_length_matches(record_offset, record); + } + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 1e884eb..485459a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -29,6 +29,7 @@ mod catalog_entry_decode_error; mod catalog_entry_decode_error_display; mod catalog_entry_decoder; mod catalog_entry_fields; +mod catalog_entry_plan; mod catalog_entry_sequence; mod catalog_header_decoder; mod catalog_header_encoding; diff --git a/tests/catalog_locations.rs b/tests/catalog_locations.rs index 8160e68..c9070e9 100644 --- a/tests/catalog_locations.rs +++ b/tests/catalog_locations.rs @@ -111,6 +111,19 @@ fn catalog_location_must_equal_one_discovered_top_level_record_span() -> Result< Ok(()) } +#[test] +fn catalog_admission_does_not_rescan_segment_records_per_entry() -> Result<(), Box> { + let admission = include_str!("../src/adapters/catalog_admission.rs"); + let plan = include_str!("../src/adapters/catalog_entry_plan.rs"); + if admission.contains(".record_cursor()") { + return Err("catalog entry iteration owns a segment record cursor".into()); + } + if plan.matches(".record_cursor()").count() != 1 || !plan.contains("entries.chunk_by_mut") { + return Err("physical entry planning does not own one grouped cursor site".into()); + } + Ok(()) +} + const fn maximum_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } diff --git a/xtask/tests/segment_store_protocol_contract.rs b/xtask/tests/segment_store_protocol_contract.rs index 5b70367..bc1cb95 100644 --- a/xtask/tests/segment_store_protocol_contract.rs +++ b/xtask/tests/segment_store_protocol_contract.rs @@ -99,7 +99,8 @@ fn durable_protocol_freezes_every_cross_cutting_law() { #[test] fn catalog_locations_name_only_top_level_segment_records() { for required in [ - "scans the complete segment grammar from byte 64", + "scans each referenced\nsegment exactly once", + "Each scan runs from byte 64", "must equal one discovered", "top-level record span", "record header, payload, checksum", From 8fe1c5a3beeb758ec8260a840f7d671ab53f710f Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:07:15 -0700 Subject: [PATCH 19/31] Refuse unreferenced catalog segments --- CHANGELOG.md | 5 +++-- docs/formats/segment-store-v1/catalog.md | 4 +++- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/catalog_admission.rs | 2 +- src/adapters/catalog_admission_error.rs | 5 +++++ .../catalog_admission_error_display.rs | 3 +++ src/adapters/catalog_entry_plan.rs | 22 ++++++++++++++++++- tests/catalog_locations/refusal_laws.rs | 19 ++++++++++++++++ 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b9a044..792e429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -186,8 +186,9 @@ after its public API and format compatibility policies are established. - Checked catalog generations; canonical catalog and publication-head codecs; exact logical-record-to-segment admission with one bounded physical lookup - plan and one scan per referenced segment; deterministic successor proofs; - immutable reader snapshots; and `BTreeMap` transition-model evidence for + plan, one scan per referenced segment, and refusal of every unreferenced + caller-supplied segment; deterministic successor proofs; immutable reader + snapshots; and `BTreeMap` transition-model evidence for `keep.segment-store/v1`. - Blocking `FilesystemCatalogPublisher` publication under a persistent kernel-managed writer lock, with pinned directory capabilities, diff --git a/docs/formats/segment-store-v1/catalog.md b/docs/formats/segment-store-v1/catalog.md index 2e16e25..74eff23 100644 --- a/docs/formats/segment-store-v1/catalog.md +++ b/docs/formats/segment-store-v1/catalog.md @@ -76,7 +76,9 @@ payload_length + 144 = record_length The named segment must exist at the digest-derived immutable-pool name and verify completely. The record at the declared span must reproduce the entry's -kind, identity, lengths, and checksum exactly. +kind, identity, lengths, and checksum exactly. A supplied admitted segment +must be named by at least one entry; admission refuses extra unreferenced +segments. Catalog admission creates one bounded plan over the declared entries, groups that plan by segment digest and physical offset, and scans each referenced diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index f44bdd1..da902ca 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -69,7 +69,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-001` | `CatalogGeneration` admits positive values and refuses overflow when deriving a successor | Checked scalar model | `tests/catalog_generation.rs` | Implemented in #16 | | `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes | Independent golden corpus | `tests/catalog.rs`, `tests/publication_head.rs` | Implemented in #16 | | `KEEP-CATALOG-003` | Catalog entries are sorted by logical identity and duplicate keys are refused independently of input order | Ordered reference map | `tests/catalog_ordering.rs` | Implemented in #16 | -| `KEEP-CATALOG-004` | Every admitted catalog location equals a verified top-level record span in the exact named segment | Segment parser and golden artifacts | `tests/catalog_locations.rs` | Implemented in #16 | +| `KEEP-CATALOG-004` | Every admitted catalog location equals a verified top-level record span in the exact named segment; every supplied segment is referenced and each referenced segment is scanned once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | | `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | diff --git a/src/adapters/catalog_admission.rs b/src/adapters/catalog_admission.rs index a5bafd1..eb22789 100644 --- a/src/adapters/catalog_admission.rs +++ b/src/adapters/catalog_admission.rs @@ -24,7 +24,7 @@ pub(super) fn admit<'catalog, 'records>( } let segment_index = index_segments(segments)?; let mut plan = plan_entries(catalog, &segment_index, requested)?; - catalog_entry_plan::bind(&mut plan)?; + catalog_entry_plan::bind(&mut plan, &segment_index)?; plan.sort_unstable_by_key(CatalogEntryPlan::ordinal); let mut bindings = Vec::new(); bindings diff --git a/src/adapters/catalog_admission_error.rs b/src/adapters/catalog_admission_error.rs index bc8eee2..3733478 100644 --- a/src/adapters/catalog_admission_error.rs +++ b/src/adapters/catalog_admission_error.rs @@ -46,6 +46,11 @@ pub enum CatalogAdmissionError { /// Required physical segment coordinate. digest: SegmentDigest, }, + /// Caller input supplied a segment absent from every catalog entry. + UnreferencedSegment { + /// Unreferenced physical segment coordinate. + digest: SegmentDigest, + }, /// Revalidating an admitted segment's immutable records failed. Segment { /// Physical segment being scanned. diff --git a/src/adapters/catalog_admission_error_display.rs b/src/adapters/catalog_admission_error_display.rs index 7370723..6f8f34e 100644 --- a/src/adapters/catalog_admission_error_display.rs +++ b/src/adapters/catalog_admission_error_display.rs @@ -31,6 +31,9 @@ impl fmt::Display for CatalogAdmissionError { formatter.write_str("duplicate admitted segment digest") } Self::MissingSegment { .. } => formatter.write_str("catalog segment is missing"), + Self::UnreferencedSegment { .. } => { + formatter.write_str("admitted segment is not referenced by the catalog") + } Self::Segment { source, .. } => { write!(formatter, "catalog segment revalidation failed: {source}") } diff --git a/src/adapters/catalog_entry_plan.rs b/src/adapters/catalog_entry_plan.rs index 3520836..40a5397 100644 --- a/src/adapters/catalog_entry_plan.rs +++ b/src/adapters/catalog_entry_plan.rs @@ -68,8 +68,12 @@ impl<'segments, 'records> CatalogEntryPlan<'segments, 'records> { } } -pub(super) fn bind(entries: &mut [CatalogEntryPlan<'_, '_>]) -> Result<(), CatalogAdmissionError> { +pub(super) fn bind( + entries: &mut [CatalogEntryPlan<'_, '_>], + segments: &[&AdmittedSegment<'_>], +) -> Result<(), CatalogAdmissionError> { entries.sort_unstable_by_key(CatalogEntryPlan::physical_order); + refuse_unreferenced_segments(entries, segments)?; for group in entries.chunk_by_mut(|first, second| first.segment_digest() == second.segment_digest()) { @@ -78,6 +82,22 @@ pub(super) fn bind(entries: &mut [CatalogEntryPlan<'_, '_>]) -> Result<(), Catal Ok(()) } +fn refuse_unreferenced_segments( + entries: &[CatalogEntryPlan<'_, '_>], + segments: &[&AdmittedSegment<'_>], +) -> Result<(), CatalogAdmissionError> { + for segment in segments { + let digest = segment.digest(); + if entries + .binary_search_by_key(&digest, CatalogEntryPlan::segment_digest) + .is_err() + { + return Err(CatalogAdmissionError::UnreferencedSegment { digest }); + } + } + Ok(()) +} + fn bind_segment(entries: &mut [CatalogEntryPlan<'_, '_>]) -> Result<(), CatalogAdmissionError> { let Some(first) = entries.first() else { return Ok(()); diff --git a/tests/catalog_locations/refusal_laws.rs b/tests/catalog_locations/refusal_laws.rs index 8fba634..a70236e 100644 --- a/tests/catalog_locations/refusal_laws.rs +++ b/tests/catalog_locations/refusal_laws.rs @@ -106,6 +106,25 @@ fn segment_input_is_bounded_and_duplicate_free() -> Result<(), Box> { Ok(()) } +#[test] +fn catalog_refuses_every_unreferenced_admitted_segment() -> Result<(), Box> { + let catalog_bytes = fixture(BUNDLE_CATALOG_HEX)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + let referenced_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let unreferenced_bytes = fixture(SEGMENT_HEX)?; + let referenced = AdmittedSegment::decode(&referenced_bytes, maximum_policy())?; + let unreferenced = AdmittedSegment::decode(&unreferenced_bytes, maximum_policy())?; + let error = require_error( + catalog.admit(&[referenced, unreferenced]), + "unreferenced physical segment input was admitted", + )?; + assert!(matches!( + error, + CatalogAdmissionError::UnreferencedSegment { .. } + )); + Ok(()) +} + fn replace_byte(target: &mut [u8], offset: usize) -> Result<(), Box> { let byte = target .get_mut(offset) From ddb52430a73d70b3861a02b597b6e4902b0180bc Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:10:58 -0700 Subject: [PATCH 20/31] Refuse unreferenced catalog inputs --- CHANGELOG.md | 4 ++-- docs/formats/segment-store-v1/catalog.md | 6 +++--- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/catalog_encode_error.rs | 9 +++++++++ src/adapters/catalog_encoder.rs | 5 +++++ tests/catalog_encoding.rs | 18 ++++++++++++++++++ 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 792e429..7399b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,8 +187,8 @@ after its public API and format compatibility policies are established. - Checked catalog generations; canonical catalog and publication-head codecs; exact logical-record-to-segment admission with one bounded physical lookup plan, one scan per referenced segment, and refusal of every unreferenced - caller-supplied segment; deterministic successor proofs; immutable reader - snapshots; and `BTreeMap` transition-model evidence for + caller-supplied segment during construction or admission; deterministic + successor proofs; immutable reader snapshots; and `BTreeMap` transition-model evidence for `keep.segment-store/v1`. - Blocking `FilesystemCatalogPublisher` publication under a persistent kernel-managed writer lock, with pinned directory capabilities, diff --git a/docs/formats/segment-store-v1/catalog.md b/docs/formats/segment-store-v1/catalog.md index 74eff23..3a612bd 100644 --- a/docs/formats/segment-store-v1/catalog.md +++ b/docs/formats/segment-store-v1/catalog.md @@ -76,9 +76,9 @@ payload_length + 144 = record_length The named segment must exist at the digest-derived immutable-pool name and verify completely. The record at the declared span must reproduce the entry's -kind, identity, lengths, and checksum exactly. A supplied admitted segment -must be named by at least one entry; admission refuses extra unreferenced -segments. +kind, identity, lengths, and checksum exactly. Every supplied admitted segment +must be named by at least one entry. Catalog construction refuses zero-record +segments, and admission refuses extra unreferenced segments. Catalog admission creates one bounded plan over the declared entries, groups that plan by segment digest and physical offset, and scans each referenced diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index da902ca..e6d0c4d 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -69,7 +69,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-001` | `CatalogGeneration` admits positive values and refuses overflow when deriving a successor | Checked scalar model | `tests/catalog_generation.rs` | Implemented in #16 | | `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes | Independent golden corpus | `tests/catalog.rs`, `tests/publication_head.rs` | Implemented in #16 | | `KEEP-CATALOG-003` | Catalog entries are sorted by logical identity and duplicate keys are refused independently of input order | Ordered reference map | `tests/catalog_ordering.rs` | Implemented in #16 | -| `KEEP-CATALOG-004` | Every admitted catalog location equals a verified top-level record span in the exact named segment; every supplied segment is referenced and each referenced segment is scanned once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_locations.rs` | Implemented in #16 | +| `KEEP-CATALOG-004` | Every catalog location equals a verified top-level record span in the exact named segment; construction and admission require every supplied segment to be referenced, and admission scans each referenced segment once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_encoding.rs`, `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | | `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | diff --git a/src/adapters/catalog_encode_error.rs b/src/adapters/catalog_encode_error.rs index 0513fa3..1e2d7ea 100644 --- a/src/adapters/catalog_encode_error.rs +++ b/src/adapters/catalog_encode_error.rs @@ -20,6 +20,11 @@ pub enum CatalogEncodeError { /// Later generation being encoded. generation: CatalogGeneration, }, + /// A supplied segment has no record that the catalog can reference. + UnreferencedSegment { + /// Exact unreferenced physical segment. + segment_digest: SegmentDigest, + }, /// Summing admitted segment record counts overflowed. EntryCountArithmetic, /// The aggregate record count exceeded the format bound. @@ -76,6 +81,9 @@ impl fmt::Display for CatalogEncodeError { "catalog generation {} requires a predecessor", generation.get() ), + Self::UnreferencedSegment { .. } => { + formatter.write_str("catalog input contains an unreferenced segment") + } Self::EntryCountArithmetic => formatter.write_str("catalog entry count overflowed"), Self::EntryCountOutOfBounds { maximum, observed } => write!( formatter, @@ -114,6 +122,7 @@ impl Error for CatalogEncodeError { Self::Segment { source, .. } => Some(source), Self::UnexpectedPredecessor { .. } | Self::MissingPredecessor { .. } + | Self::UnreferencedSegment { .. } | Self::EntryCountArithmetic | Self::EntryCountOutOfBounds { .. } | Self::HostLength { .. } diff --git a/src/adapters/catalog_encoder.rs b/src/adapters/catalog_encoder.rs index 4722bb8..fa2dd5e 100644 --- a/src/adapters/catalog_encoder.rs +++ b/src/adapters/catalog_encoder.rs @@ -50,6 +50,11 @@ fn validate_predecessor( fn entry_count(segments: &[AdmittedSegment<'_>]) -> Result { let mut count = 0_u64; for segment in segments { + if segment.record_count() == 0 { + return Err(CatalogEncodeError::UnreferencedSegment { + segment_digest: segment.digest(), + }); + } count = count .checked_add(u64::from(segment.record_count())) .ok_or(CatalogEncodeError::EntryCountArithmetic)?; diff --git a/tests/catalog_encoding.rs b/tests/catalog_encoding.rs index 025bbc8..a41941e 100644 --- a/tests/catalog_encoding.rs +++ b/tests/catalog_encoding.rs @@ -13,6 +13,7 @@ use support::{decode_hex, require_error}; const ONE_ZERO_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/empty-segment.hex"); const BUNDLE_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); const ONE_ZERO_CATALOG_HEX: &str = @@ -105,6 +106,23 @@ fn duplicate_logical_records_are_refused_before_emission() -> Result<(), Box Result<(), Box> { + let empty_bytes = fixture(EMPTY_SEGMENT_HEX)?; + let segments = [admitted_segment(&empty_bytes)?]; + let expected = segments[0].digest(); + let error = require_error( + CanonicalCatalog::from_segments(generation(1)?, None, &segments), + "empty physical segment was silently omitted from the catalog", + )?; + assert!(matches!( + error, + CatalogEncodeError::UnreferencedSegment { segment_digest } + if segment_digest == expected + )); + Ok(()) +} + fn assert_head(catalog_hex: &str, head_hex: &str) -> Result<(), Box> { let catalog_bytes = fixture(catalog_hex)?; let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; From 5ab28cd4b7ee1e937de6400200489c0a7a5050fe Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:13:45 -0700 Subject: [PATCH 21/31] Document exact catalog input contracts --- src/adapters/canonical_catalog.rs | 11 ++++++----- src/adapters/checksummed_catalog.rs | 14 ++++++++------ tests/catalog_encoding.rs | 13 +++++++++++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/adapters/canonical_catalog.rs b/src/adapters/canonical_catalog.rs index 16a7750..94467a5 100644 --- a/src/adapters/canonical_catalog.rs +++ b/src/adapters/canonical_catalog.rs @@ -7,9 +7,10 @@ use crate::{CatalogDigest, CatalogGeneration}; /// Owned canonical version-1 catalog bytes. /// /// Construction derives physical record coordinates from fully admitted -/// segments, sorts entries by logical identity, and refuses duplicates. The -/// complete catalog is materialized in memory with the version-1 entry-count -/// and byte-length bounds enforced before allocation. +/// segments, sorts entries by logical identity, and refuses duplicate records +/// or an unreferenced segment. The complete catalog is materialized in memory +/// with the version-1 entry-count and byte-length bounds enforced before +/// allocation. #[must_use] #[derive(Debug, Eq, PartialEq)] pub struct CanonicalCatalog { @@ -24,8 +25,8 @@ impl CanonicalCatalog { /// # Errors /// /// Returns [`CatalogEncodeError`] for an invalid predecessor law, checked - /// count or length refusal, allocation failure, failed immutable segment - /// revalidation, or duplicate logical identity. + /// count or length refusal, unreferenced segment, allocation failure, + /// failed immutable segment revalidation, or duplicate logical identity. pub fn from_segments( generation: CatalogGeneration, previous_catalog_digest: Option, diff --git a/src/adapters/checksummed_catalog.rs b/src/adapters/checksummed_catalog.rs index a4e097b..c86e675 100644 --- a/src/adapters/checksummed_catalog.rs +++ b/src/adapters/checksummed_catalog.rs @@ -67,15 +67,17 @@ impl<'a> ChecksummedCatalog<'a> { /// Binds every logical entry to one exact top-level admitted segment record. /// /// This operation performs no I/O. It temporarily allocates one sorted - /// borrowed segment index and retains one logical record binding per entry. - /// Both allocations are bounded by caller input or the verified catalog - /// entry count. + /// borrowed segment index and one physical entry plan, then retains one + /// logical record binding per entry. All three allocations are bounded by + /// caller input or the verified catalog entry count. Each referenced + /// segment is scanned once. /// /// # Errors /// - /// Returns [`CatalogAdmissionError`] for allocation refusal, duplicate or - /// missing segments, failed immutable revalidation, interior locations, or - /// disagreement between catalog fields and the selected record. + /// Returns [`CatalogAdmissionError`] for allocation refusal, duplicate, + /// missing, or unreferenced segments, failed immutable revalidation, + /// interior locations, or disagreement between catalog fields and the + /// selected record. pub fn admit<'records>( self, segments: &[AdmittedSegment<'records>], diff --git a/tests/catalog_encoding.rs b/tests/catalog_encoding.rs index a41941e..d4261c5 100644 --- a/tests/catalog_encoding.rs +++ b/tests/catalog_encoding.rs @@ -123,6 +123,19 @@ fn catalog_encoding_refuses_an_unreferenced_empty_segment() -> Result<(), Box Result<(), Box> { + let canonical = include_str!("../src/adapters/canonical_catalog.rs"); + let checksummed = include_str!("../src/adapters/checksummed_catalog.rs"); + if !canonical.contains("unreferenced segment") { + return Err("canonical catalog rustdoc omits its exact-input refusal".into()); + } + if !checksummed.contains("entry plan") || !checksummed.contains("unreferenced segments") { + return Err("catalog admission rustdoc omits its current resource contract".into()); + } + Ok(()) +} + fn assert_head(catalog_hex: &str, head_hex: &str) -> Result<(), Box> { let catalog_bytes = fixture(catalog_hex)?; let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; From f0b1527f8918ea4d8e5427439a306a58fa43a325 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:20:58 -0700 Subject: [PATCH 22/31] Make durable publication retries idempotent --- CHANGELOG.md | 3 ++ docs/formats/segment-store-v1/publication.md | 12 +++-- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/catalog_publication.rs | 23 +++++++--- src/adapters/catalog_publication_execution.rs | 26 +++++++---- src/adapters/catalog_publication_outcome.rs | 11 +++++ src/adapters/catalog_publication_readiness.rs | 11 +++++ src/adapters/catalog_publication_receipt.rs | 25 +++++++++-- src/adapters/catalog_publication_storage.rs | 14 ++++-- src/adapters/filesystem_catalog_current.rs | 19 +++++--- src/adapters/filesystem_catalog_storage.rs | 14 +++--- src/adapters/mod.rs | 4 ++ src/lib.rs | 33 +++++++------- tests/catalog_filesystem_publication.rs | 44 +++++++++++++++++-- .../refusal_laws.rs | 5 ++- tests/catalog_publication/preflight_laws.rs | 32 +++++++++++++- .../catalog_publication/recording_storage.rs | 24 ++++++++-- 17 files changed, 237 insertions(+), 65 deletions(-) create mode 100644 src/adapters/catalog_publication_outcome.rs create mode 100644 src/adapters/catalog_publication_readiness.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7399b42..3355d3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -198,6 +198,9 @@ after its public API and format compatibility policies are established. refusal before mutation. New segment publication consumes a handle-free `ClosedSegment` proof before any immutable-pool link, and publisher teardown closes every retained writable handle before releasing writer authority. + Retry of an already-current complete candidate re-synchronizes the root and + returns an explicit `CatalogPublicationOutcome::AlreadyPublished` receipt + without repeating publication mutations. - Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact checksummed head, catalog, and segment coordinates; refuses symbolic links, nonregular files, malformed or conflicting bytes, dangling entries, and diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index 0592360..34e2996 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -162,9 +162,10 @@ The writer starts with an expected current generation and catalog digest. It acquires the lock, validates the current head and catalog again, and refuses stale expectations before creating a stage. If the current verified head already equals the complete proposed generation, catalog length, and catalog -digest, retry returns an explicit already-published receipt after -synchronizing the root directory. A different observed generation or digest -is a stale-generation refusal. +digest, retry returns +`CatalogPublicationOutcome::AlreadyPublished` after synchronizing the root +directory. A different observed generation or digest is a stale-generation +refusal. Every write handles short writes and interruption. Every flush, file sync, hard link, unlink, head replacement, and directory sync is explicit and @@ -227,7 +228,10 @@ until the publication head names it. atomically replace `HEAD` with `head.next` (`KEEP-CRASH-025`). 6. Synchronize the store root (`KEEP-CRASH-026`). -Only completion of step 6 returns a `#[must_use]` publication receipt. +Only completion of step 6 returns a +`CatalogPublicationOutcome::Published` receipt for a new publication. An +already-current retry returns `CatalogPublicationOutcome::AlreadyPublished` +only after complete candidate revalidation and a fresh root synchronization. An existing `head.next` always routes through recovery before step 1. The writer never truncates, replaces, or silently removes it to make the exclusive diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index e6d0c4d..c13f47b 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -73,7 +73,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | | `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | -| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented file and directory synchronization order | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | +| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | diff --git a/src/adapters/catalog_publication.rs b/src/adapters/catalog_publication.rs index 08f03dc..499594f 100644 --- a/src/adapters/catalog_publication.rs +++ b/src/adapters/catalog_publication.rs @@ -3,16 +3,18 @@ use super::catalog_publication_expectation::ExpectedCurrentCatalog; use super::{ AdmittedCatalog, AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogTransitionError, ChecksummedPublicationHead, - SegmentPublication, catalog_publication_execution, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogTransitionError, + ChecksummedPublicationHead, SegmentPublication, catalog_publication_execution, }; /// Publishes one fully admitted canonical catalog generation. /// /// All catalog, segment, and head relationships are verified before the first -/// storage transition. A successful receipt is returned only after atomic head -/// replacement and root-directory synchronization. +/// storage transition. A new publication returns only after atomic head +/// replacement and root-directory synchronization. If the complete candidate +/// is already current, the retry performs no publication mutation, +/// re-synchronizes the root, and returns an explicit already-published outcome. /// /// # Errors /// @@ -40,13 +42,20 @@ pub fn publish_catalog_generation( let snapshot = checked_head .admit(admitted) .map_err(|source| CatalogPublicationError::SnapshotAdmission { source })?; - catalog_publication_execution::execute_current(storage, expectation)?; + let readiness = + catalog_publication_execution::execute_current(storage, expectation, &snapshot)?; + if readiness == CatalogPublicationReadiness::AlreadyPublished { + return Ok(CatalogPublicationReceipt::already_published( + snapshot.generation(), + snapshot.catalog_digest(), + )); + } if let Some(segment) = segment.into_admitted() { catalog_publication_execution::execute_segment(storage, segment)?; } catalog_publication_execution::execute_catalog(storage, catalog, checksummed)?; catalog_publication_execution::execute_head(storage, &head, &snapshot)?; - Ok(CatalogPublicationReceipt::synchronized( + Ok(CatalogPublicationReceipt::published( snapshot.generation(), snapshot.catalog_digest(), )) diff --git a/src/adapters/catalog_publication_execution.rs b/src/adapters/catalog_publication_execution.rs index 28a596f..766327c 100644 --- a/src/adapters/catalog_publication_execution.rs +++ b/src/adapters/catalog_publication_execution.rs @@ -4,18 +4,26 @@ use std::io; use super::{ CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationPhase, CatalogPublicationStorage, - CatalogSnapshot, ChecksummedCatalog, + CatalogPublicationExpectation, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, }; pub(super) fn execute_current( storage: &mut impl CatalogPublicationStorage, expectation: CatalogPublicationExpectation, -) -> Result<(), CatalogPublicationError> { - phase( + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result { + let readiness = phase( CatalogPublicationPhase::VerifyCurrent, - storage.verify_current(expectation), - ) + storage.verify_current(expectation, candidate), + )?; + if readiness == CatalogPublicationReadiness::AlreadyPublished { + phase( + CatalogPublicationPhase::SynchronizeRoot, + storage.synchronize_root(), + )?; + } + Ok(readiness) } pub(super) fn execute_segment( @@ -113,9 +121,9 @@ pub(super) fn execute_head( ) } -fn phase( +fn phase( phase: CatalogPublicationPhase, - result: io::Result<()>, -) -> Result<(), CatalogPublicationError> { + result: io::Result, +) -> Result { result.map_err(|source| CatalogPublicationError::storage(phase, source)) } diff --git a/src/adapters/catalog_publication_outcome.rs b/src/adapters/catalog_publication_outcome.rs new file mode 100644 index 0000000..fc6d482 --- /dev/null +++ b/src/adapters/catalog_publication_outcome.rs @@ -0,0 +1,11 @@ +//! Final outcome of one synchronized catalog publication attempt. + +/// Whether this call published a generation or verified an earlier completion. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogPublicationOutcome { + /// This call completed the catalog and head publication protocol. + Published, + /// The proposed generation was already current and was durably reverified. + AlreadyPublished, +} diff --git a/src/adapters/catalog_publication_readiness.rs b/src/adapters/catalog_publication_readiness.rs new file mode 100644 index 0000000..849789d --- /dev/null +++ b/src/adapters/catalog_publication_readiness.rs @@ -0,0 +1,11 @@ +//! Storage decision after current catalog state verification. + +/// Writer-locked decision for a fully preflighted catalog candidate. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogPublicationReadiness { + /// The expected predecessor is current, so publication may proceed. + Ready, + /// The complete proposed generation is already current. + AlreadyPublished, +} diff --git a/src/adapters/catalog_publication_receipt.rs b/src/adapters/catalog_publication_receipt.rs index 5656ed3..e6f4022 100644 --- a/src/adapters/catalog_publication_receipt.rs +++ b/src/adapters/catalog_publication_receipt.rs @@ -1,13 +1,15 @@ -//! Consequential receipt for one fully synchronized catalog generation. +//! Consequential receipt for one synchronized catalog publication attempt. +use super::CatalogPublicationOutcome; use crate::{CatalogDigest, CatalogGeneration}; -/// Proof that publication reached root-directory synchronization. +/// Proof that a candidate became or remained current through root synchronization. #[must_use] #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct CatalogPublicationReceipt { generation: CatalogGeneration, catalog_digest: CatalogDigest, + outcome: CatalogPublicationOutcome, } impl CatalogPublicationReceipt { @@ -21,13 +23,30 @@ impl CatalogPublicationReceipt { self.catalog_digest } - pub(super) const fn synchronized( + /// Returns whether this call published or reverified the generation. + pub const fn outcome(self) -> CatalogPublicationOutcome { + self.outcome + } + + pub(super) const fn published( + generation: CatalogGeneration, + catalog_digest: CatalogDigest, + ) -> Self { + Self { + generation, + catalog_digest, + outcome: CatalogPublicationOutcome::Published, + } + } + + pub(super) const fn already_published( generation: CatalogGeneration, catalog_digest: CatalogDigest, ) -> Self { Self { generation, catalog_digest, + outcome: CatalogPublicationOutcome::AlreadyPublished, } } } diff --git a/src/adapters/catalog_publication_storage.rs b/src/adapters/catalog_publication_storage.rs index ce04803..4dae315 100644 --- a/src/adapters/catalog_publication_storage.rs +++ b/src/adapters/catalog_publication_storage.rs @@ -4,7 +4,7 @@ use std::io; use super::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, - CatalogSnapshot, ChecksummedCatalog, + CatalogPublicationReadiness, CatalogSnapshot, ChecksummedCatalog, }; /// Blocking filesystem capabilities for the catalog publication protocol. @@ -14,12 +14,20 @@ use super::{ /// transitions and must not combine later transitions or report success before /// the named durability or verification obligation is satisfied. pub trait CatalogPublicationStorage { - /// Reopens and verifies the exact expected current publication state. + /// Reopens and verifies the expected predecessor or complete candidate. + /// + /// Returns [`CatalogPublicationReadiness::Ready`] only when `expected` is + /// current. Returns [`CatalogPublicationReadiness::AlreadyPublished`] only + /// when the exact generation and digest in `candidate` are current. /// /// # Errors /// /// Returns the exact current-state verification failure. - fn verify_current(&mut self, expected: CatalogPublicationExpectation) -> io::Result<()>; + fn verify_current( + &mut self, + expected: CatalogPublicationExpectation, + candidate: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result; /// Links the exact sealed stage without replacing an immutable pool entry. /// diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs index ed92869..4e33d96 100644 --- a/src/adapters/filesystem_catalog_current.rs +++ b/src/adapters/filesystem_catalog_current.rs @@ -3,15 +3,16 @@ use std::io; use super::{ - CatalogPublicationExpectation, CatalogRestartError, CatalogRestartPhase, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, catalog_restart_loader, - filesystem_catalog_artifact, + CatalogPublicationExpectation, CatalogPublicationReadiness, CatalogRestartError, + CatalogRestartPhase, CatalogSnapshot, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, catalog_restart_loader, filesystem_catalog_artifact, }; pub(super) fn verify( publisher: &FilesystemCatalogPublisher, expected: CatalogPublicationExpectation, -) -> io::Result<()> { + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> io::Result { require_no_next_head(publisher)?; match catalog_restart_loader::load_from_directory( &publisher.root, @@ -24,7 +25,11 @@ pub(super) fn verify( if expected.current_generation() == observed_generation && expected.current_catalog_digest() == observed_digest { - Ok(()) + Ok(CatalogPublicationReadiness::Ready) + } else if observed.generation() == candidate.generation() + && observed.catalog_digest() == candidate.catalog_digest() + { + Ok(CatalogPublicationReadiness::AlreadyPublished) } else { Err(filesystem_catalog_artifact::invalid_data( FilesystemCatalogPublicationError::CurrentState { @@ -36,7 +41,9 @@ pub(super) fn verify( )) } } - Err(source) if head_is_absent(&source) && expected.current_generation().is_none() => Ok(()), + Err(source) if head_is_absent(&source) && expected.current_generation().is_none() => { + Ok(CatalogPublicationReadiness::Ready) + } Err(source) => Err(filesystem_catalog_artifact::invalid_data(source)), } } diff --git a/src/adapters/filesystem_catalog_storage.rs b/src/adapters/filesystem_catalog_storage.rs index 80f8385..b2a8d54 100644 --- a/src/adapters/filesystem_catalog_storage.rs +++ b/src/adapters/filesystem_catalog_storage.rs @@ -4,14 +4,18 @@ use std::io; use super::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, - CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, FilesystemCatalogPublisher, - filesystem_catalog_catalog, filesystem_catalog_current, filesystem_catalog_head, - filesystem_catalog_segment, + CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, + FilesystemCatalogPublisher, filesystem_catalog_catalog, filesystem_catalog_current, + filesystem_catalog_head, filesystem_catalog_segment, }; impl CatalogPublicationStorage for FilesystemCatalogPublisher { - fn verify_current(&mut self, expected: CatalogPublicationExpectation) -> io::Result<()> { - filesystem_catalog_current::verify(self, expected) + fn verify_current( + &mut self, + expected: CatalogPublicationExpectation, + candidate: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result { + filesystem_catalog_current::verify(self, expected, candidate) } fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 485459a..e54ffed 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -38,7 +38,9 @@ mod catalog_publication; mod catalog_publication_error; mod catalog_publication_execution; mod catalog_publication_expectation; +mod catalog_publication_outcome; mod catalog_publication_phase; +mod catalog_publication_readiness; mod catalog_publication_receipt; mod catalog_publication_storage; mod catalog_record_binding; @@ -171,7 +173,9 @@ pub use catalog_entry_decode_error::CatalogEntryDecodeError; pub use catalog_publication::publish_catalog_generation; pub use catalog_publication_error::CatalogPublicationError; pub use catalog_publication_expectation::CatalogPublicationExpectation; +pub use catalog_publication_outcome::CatalogPublicationOutcome; pub use catalog_publication_phase::CatalogPublicationPhase; +pub use catalog_publication_readiness::CatalogPublicationReadiness; pub use catalog_publication_receipt::CatalogPublicationReceipt; pub use catalog_publication_storage::CatalogPublicationStorage; pub use catalog_restart_artifact::CatalogRestartArtifact; diff --git a/src/lib.rs b/src/lib.rs index 41b23fb..54393e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,22 +30,23 @@ pub use adapters::{ BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationPhase, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - PublicationHeadDecodeError, SealedSegment, SegmentDigest, SegmentDurabilityPhase, - SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, - SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - WriterLockAcquireError, WriterLockAcquirePhase, publish_catalog_generation, + CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, + CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, + CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, PublicationHeadDecodeError, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, + SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, + SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, + SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, + SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, + StorageProfileIdParseError, WriterLockAcquireError, WriterLockAcquirePhase, + publish_catalog_generation, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index 75883d7..e3ddc4d 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -12,10 +12,10 @@ use std::path::{Path, PathBuf}; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, - CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, ClosedSegment, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemWriterLock, LayoutEntryLimit, - SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, - publish_catalog_generation, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogRestartByteLimit, + CatalogRestartPolicy, ClosedSegment, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemWriterLock, LayoutEntryLimit, SegmentPublication, SegmentReadPolicy, + SegmentRecordLimit, StagedSegment, publish_catalog_generation, }; use sandbox::TestDirectory; use support::decode_hex; @@ -51,6 +51,7 @@ fn successful_publication_materializes_only_the_exact_durable_view() -> Result<( )?; drop(publisher); + assert_eq!(receipt.outcome(), CatalogPublicationOutcome::Published); assert_eq!(receipt.generation().get(), 1); assert_eq!(fs::read(store.path().join("HEAD"))?, fixture(HEAD_HEX)?); assert_eq!(fs::read(store.catalog_path())?, fixture(CATALOG_HEX)?); @@ -63,6 +64,41 @@ fn successful_publication_materializes_only_the_exact_durable_view() -> Result<( store.remove() } +#[test] +fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-retry")?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let first = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::one(closed, &segments[0])?, + &catalog, + &segments, + )?; + drop(publisher); + + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let retry = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::none(), + &catalog, + &segments, + )?; + + assert_eq!(retry.generation(), first.generation()); + assert_eq!(retry.catalog_digest(), first.catalog_digest()); + assert_eq!(retry.outcome(), CatalogPublicationOutcome::AlreadyPublished); + drop(publisher); + store.remove() +} + struct StoreFixture { sandbox: TestDirectory, catalog_path: PathBuf, diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index fe437b1..186a855 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -71,6 +71,7 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< )?; drop(publisher); + let stale_candidate = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &[])?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; let error = require_error( @@ -78,8 +79,8 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< &mut publisher, CatalogPublicationExpectation::uninitialized(), SegmentPublication::none(), - &catalog, - &segments, + &stale_candidate, + &[], ), "stale uninitialized expectation was accepted", )?; diff --git a/tests/catalog_publication/preflight_laws.rs b/tests/catalog_publication/preflight_laws.rs index a83aa1d..1e7348b 100644 --- a/tests/catalog_publication/preflight_laws.rs +++ b/tests/catalog_publication/preflight_laws.rs @@ -4,8 +4,9 @@ use std::error::Error; use keep::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, - CatalogPublicationError, CatalogPublicationExpectation, CatalogTransitionError, - ChecksummedPublicationHead, SegmentPublication, publish_catalog_generation, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogTransitionError, ChecksummedPublicationHead, + SegmentPublication, publish_catalog_generation, }; use super::recording_storage::RecordingStorage; @@ -66,6 +67,33 @@ fn catalog_location_refusal_precedes_every_storage_call() -> Result<(), Box Result<(), Box> { + let bytes = fixture(SEGMENT_HEX)?; + let publication = publication_fixture(&bytes)?; + let mut storage = RecordingStorage::already_published(); + let receipt = publish_catalog_generation( + &mut storage, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::none(), + &publication.catalog, + &publication.segments, + )?; + + assert_eq!( + receipt.outcome(), + CatalogPublicationOutcome::AlreadyPublished + ); + assert_eq!( + storage.observed(), + &[ + CatalogPublicationPhase::VerifyCurrent, + CatalogPublicationPhase::SynchronizeRoot, + ] + ); + Ok(()) +} + #[test] fn current_snapshot_requires_and_admits_only_its_exact_successor() -> Result<(), Box> { let bytes = fixture(SEGMENT_HEX)?; diff --git a/tests/catalog_publication/recording_storage.rs b/tests/catalog_publication/recording_storage.rs index 0ad601d..cbd613b 100644 --- a/tests/catalog_publication/recording_storage.rs +++ b/tests/catalog_publication/recording_storage.rs @@ -4,7 +4,8 @@ use std::io; use keep::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, - CatalogPublicationPhase, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationStorage, + CatalogSnapshot, ChecksummedCatalog, }; /// Exact complete publication order when one segment stage is present. @@ -37,6 +38,7 @@ pub const EXPECTED_WITH_SEGMENT: &[CatalogPublicationPhase] = &[ pub struct RecordingStorage { observed: Vec, failing_phase: Option, + readiness: CatalogPublicationReadiness, } impl RecordingStorage { @@ -45,6 +47,16 @@ impl RecordingStorage { Self { observed: Vec::new(), failing_phase: None, + readiness: CatalogPublicationReadiness::Ready, + } + } + + /// Creates a recorder that reports the complete candidate as current. + pub const fn already_published() -> Self { + Self { + observed: Vec::new(), + failing_phase: None, + readiness: CatalogPublicationReadiness::AlreadyPublished, } } @@ -53,6 +65,7 @@ impl RecordingStorage { Self { observed: Vec::new(), failing_phase: Some(phase), + readiness: CatalogPublicationReadiness::Ready, } } @@ -72,8 +85,13 @@ impl RecordingStorage { } impl CatalogPublicationStorage for RecordingStorage { - fn verify_current(&mut self, _expected: CatalogPublicationExpectation) -> io::Result<()> { - self.record(CatalogPublicationPhase::VerifyCurrent) + fn verify_current( + &mut self, + _expected: CatalogPublicationExpectation, + _candidate: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result { + self.record(CatalogPublicationPhase::VerifyCurrent)?; + Ok(self.readiness) } fn link_segment(&mut self, _segment: &AdmittedSegment<'_>) -> io::Result<()> { From 98d8e8025ae3ef0bcd8583565a911ea0b8d8a987 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:27:12 -0700 Subject: [PATCH 23/31] Refuse retained stages before publication --- CHANGELOG.md | 5 +- docs/formats/segment-store-v1/publication.md | 11 +- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/catalog_publication.rs | 2 +- src/adapters/catalog_publication_execution.rs | 5 +- src/adapters/catalog_publication_storage.rs | 7 +- src/adapters/filesystem_catalog_current.rs | 46 ++++++- .../filesystem_catalog_publication_error.rs | 10 ++ src/adapters/filesystem_catalog_storage.rs | 7 +- .../refusal_laws.rs | 125 ++++++++++++++++++ .../catalog_publication/recording_storage.rs | 3 +- 11 files changed, 204 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3355d3a..bb2293d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -200,7 +200,10 @@ after its public API and format compatibility policies are established. closes every retained writable handle before releasing writer authority. Retry of an already-current complete candidate re-synchronizes the root and returns an explicit `CatalogPublicationOutcome::AlreadyPublished` receipt - without repeating publication mutations. + without repeating publication mutations. Retained `head.next` or + `current.cat`, an unselected `current.seg`, and every fixed-name stage on an + already-current retry now refuse at current-state verification before any + publication mutation. - Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact checksummed head, catalog, and segment coordinates; refuses symbolic links, nonregular files, malformed or conflicting bytes, dangling entries, and diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index 34e2996..49977ae 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -153,8 +153,10 @@ after all canonical bytes and physical coordinates verify. Issue #16 does not implement store-root initialization or explicit recovery. A caller must supply the exact canonical directories and persistent lock file -before opening a publisher. Any retained `head.next` causes publication to -refuse before mutation and requires issue #17 recovery. +before opening a publisher. Any retained `head.next` or `current.cat`, and any +`current.seg` not owned by the selected staged segment, causes publication to +refuse before mutation and requires issue #17 recovery. An already-current +retry refuses every fixed-name stage. ## Forward publication protocol @@ -233,8 +235,9 @@ Only completion of step 6 returns a already-current retry returns `CatalogPublicationOutcome::AlreadyPublished` only after complete candidate revalidation and a fresh root synchronization. -An existing `head.next` always routes through recovery before step 1. The -writer never truncates, replaces, or silently removes it to make the exclusive +An existing `head.next` or `current.cat`, or an unselected `current.seg`, +always routes through recovery before step 1. The writer never truncates, +replaces, or silently removes retained stage evidence to make an exclusive create succeed. The normative pre-state, interrupted-state class, post-state, and recovery diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index c13f47b..9a52351 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -73,7 +73,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | | `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | -| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | +| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | diff --git a/src/adapters/catalog_publication.rs b/src/adapters/catalog_publication.rs index 499594f..d3a9b10 100644 --- a/src/adapters/catalog_publication.rs +++ b/src/adapters/catalog_publication.rs @@ -43,7 +43,7 @@ pub fn publish_catalog_generation( .admit(admitted) .map_err(|source| CatalogPublicationError::SnapshotAdmission { source })?; let readiness = - catalog_publication_execution::execute_current(storage, expectation, &snapshot)?; + catalog_publication_execution::execute_current(storage, expectation, &snapshot, &segment)?; if readiness == CatalogPublicationReadiness::AlreadyPublished { return Ok(CatalogPublicationReceipt::already_published( snapshot.generation(), diff --git a/src/adapters/catalog_publication_execution.rs b/src/adapters/catalog_publication_execution.rs index 766327c..1b5e699 100644 --- a/src/adapters/catalog_publication_execution.rs +++ b/src/adapters/catalog_publication_execution.rs @@ -5,17 +5,18 @@ use std::io; use super::{ CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationPhase, CatalogPublicationReadiness, - CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, + CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, SegmentPublication, }; pub(super) fn execute_current( storage: &mut impl CatalogPublicationStorage, expectation: CatalogPublicationExpectation, candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, ) -> Result { let readiness = phase( CatalogPublicationPhase::VerifyCurrent, - storage.verify_current(expectation, candidate), + storage.verify_current(expectation, candidate, segment), )?; if readiness == CatalogPublicationReadiness::AlreadyPublished { phase( diff --git a/src/adapters/catalog_publication_storage.rs b/src/adapters/catalog_publication_storage.rs index 4dae315..561a346 100644 --- a/src/adapters/catalog_publication_storage.rs +++ b/src/adapters/catalog_publication_storage.rs @@ -4,7 +4,7 @@ use std::io; use super::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, - CatalogPublicationReadiness, CatalogSnapshot, ChecksummedCatalog, + CatalogPublicationReadiness, CatalogSnapshot, ChecksummedCatalog, SegmentPublication, }; /// Blocking filesystem capabilities for the catalog publication protocol. @@ -18,7 +18,9 @@ pub trait CatalogPublicationStorage { /// /// Returns [`CatalogPublicationReadiness::Ready`] only when `expected` is /// current. Returns [`CatalogPublicationReadiness::AlreadyPublished`] only - /// when the exact generation and digest in `candidate` are current. + /// when the exact generation and digest in `candidate` are current. The + /// `segment` selection identifies whether a `current.seg` stage belongs to + /// this call; every other fixed-name stage is recovery state. /// /// # Errors /// @@ -27,6 +29,7 @@ pub trait CatalogPublicationStorage { &mut self, expected: CatalogPublicationExpectation, candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, ) -> io::Result; /// Links the exact sealed stage without replacing an immutable pool entry. diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs index 4e33d96..d1e238e 100644 --- a/src/adapters/filesystem_catalog_current.rs +++ b/src/adapters/filesystem_catalog_current.rs @@ -5,18 +5,24 @@ use std::io; use super::{ CatalogPublicationExpectation, CatalogPublicationReadiness, CatalogRestartError, CatalogRestartPhase, CatalogSnapshot, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, catalog_restart_loader, filesystem_catalog_artifact, + FilesystemCatalogPublisher, SegmentPublication, catalog_restart_loader, + filesystem_catalog_artifact, filesystem_catalog_publisher, }; pub(super) fn verify( publisher: &FilesystemCatalogPublisher, expected: CatalogPublicationExpectation, candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, ) -> io::Result { require_no_next_head(publisher)?; - match catalog_restart_loader::load_from_directory( + require_no_catalog_stage(publisher)?; + if segment.admitted().is_none() { + require_no_segment_stage(publisher)?; + } + let readiness = match catalog_restart_loader::load_from_directory( &publisher.root, - super::filesystem_catalog_publisher::HEAD, + filesystem_catalog_publisher::HEAD, publisher.policy, ) { Ok(observed) => { @@ -45,13 +51,17 @@ pub(super) fn verify( Ok(CatalogPublicationReadiness::Ready) } Err(source) => Err(filesystem_catalog_artifact::invalid_data(source)), + }?; + if readiness == CatalogPublicationReadiness::AlreadyPublished { + require_no_segment_stage(publisher)?; } + Ok(readiness) } fn require_no_next_head(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { match publisher .root - .symlink_metadata(super::filesystem_catalog_publisher::NEXT_HEAD) + .symlink_metadata(filesystem_catalog_publisher::NEXT_HEAD) { Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), Err(source) => Err(source), @@ -61,6 +71,34 @@ fn require_no_next_head(publisher: &FilesystemCatalogPublisher) -> io::Result<() } } +fn require_no_catalog_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + require_absent( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_CATALOG, + FilesystemCatalogPublicationError::CatalogRecoveryRequired, + ) +} + +fn require_no_segment_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + require_absent( + &publisher.staging, + filesystem_catalog_publisher::CURRENT_SEGMENT, + FilesystemCatalogPublicationError::SegmentRecoveryRequired, + ) +} + +fn require_absent( + directory: &cap_std::fs::Dir, + name: &str, + error: FilesystemCatalogPublicationError, +) -> io::Result<()> { + match directory.symlink_metadata(name) { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(source), + Ok(_metadata) => Err(filesystem_catalog_artifact::invalid_data(error)), + } +} + fn head_is_absent(error: &CatalogRestartError) -> bool { matches!( error, diff --git a/src/adapters/filesystem_catalog_publication_error.rs b/src/adapters/filesystem_catalog_publication_error.rs index 898c51c..ec54739 100644 --- a/src/adapters/filesystem_catalog_publication_error.rs +++ b/src/adapters/filesystem_catalog_publication_error.rs @@ -32,6 +32,10 @@ pub enum FilesystemCatalogPublicationError { }, /// A leftover `head.next` requires explicit recovery. HeadRecoveryRequired, + /// A leftover `staging/current.cat` requires explicit recovery. + CatalogRecoveryRequired, + /// A leftover `staging/current.seg` requires explicit recovery. + SegmentRecoveryRequired, } impl fmt::Display for FilesystemCatalogPublicationError { @@ -47,6 +51,12 @@ impl fmt::Display for FilesystemCatalogPublicationError { Self::HeadRecoveryRequired => { formatter.write_str("head.next requires explicit recovery") } + Self::CatalogRecoveryRequired => { + formatter.write_str("current.cat requires explicit recovery") + } + Self::SegmentRecoveryRequired => { + formatter.write_str("current.seg requires explicit recovery") + } } } } diff --git a/src/adapters/filesystem_catalog_storage.rs b/src/adapters/filesystem_catalog_storage.rs index b2a8d54..c6a778b 100644 --- a/src/adapters/filesystem_catalog_storage.rs +++ b/src/adapters/filesystem_catalog_storage.rs @@ -5,8 +5,8 @@ use std::io; use super::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, - FilesystemCatalogPublisher, filesystem_catalog_catalog, filesystem_catalog_current, - filesystem_catalog_head, filesystem_catalog_segment, + FilesystemCatalogPublisher, SegmentPublication, filesystem_catalog_catalog, + filesystem_catalog_current, filesystem_catalog_head, filesystem_catalog_segment, }; impl CatalogPublicationStorage for FilesystemCatalogPublisher { @@ -14,8 +14,9 @@ impl CatalogPublicationStorage for FilesystemCatalogPublisher { &mut self, expected: CatalogPublicationExpectation, candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, ) -> io::Result { - filesystem_catalog_current::verify(self, expected, candidate) + filesystem_catalog_current::verify(self, expected, candidate, segment) } fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index 186a855..1d90395 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -142,3 +142,128 @@ fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box< assert!(!store.path().join("HEAD").exists()); store.remove() } + +#[test] +fn leftover_catalog_stage_refuses_before_segment_pool_mutation() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-catalog-recovery")?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + fs::write(store.staging().join("current.cat"), b"recovery evidence")?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let error = require_error( + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::one(closed, &segments[0])?, + &catalog, + &segments, + ), + "leftover catalog stage was discovered after segment publication", + )?; + + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("catalog-stage recovery returned the wrong error".into()); + }; + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert!(matches!( + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(FilesystemCatalogPublicationError::CatalogRecoveryRequired) + )); + assert!(!store.segment_path().exists()); + assert!(store.staging().join("current.seg").exists()); + assert_eq!( + fs::read(store.staging().join("current.cat"))?, + b"recovery evidence" + ); + drop(publisher); + store.remove() +} + +#[test] +fn catalog_only_publication_refuses_a_leftover_segment_stage() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-segment-recovery")?; + let stage = store.staging().join("current.seg"); + fs::write(&stage, b"recovery evidence")?; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &[])?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let error = require_error( + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::none(), + &catalog, + &[], + ), + "catalog-only publication ignored a leftover segment stage", + )?; + + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("segment-stage recovery returned the wrong error".into()); + }; + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert!(matches!( + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(FilesystemCatalogPublicationError::SegmentRecoveryRequired) + )); + assert_eq!(fs::read(stage)?, b"recovery evidence"); + assert!(!store.path().join("HEAD").exists()); + drop(publisher); + store.remove() +} + +#[test] +fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-retry-stage")?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let _receipt = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::one(closed, &segments[0])?, + &catalog, + &segments, + )?; + drop(publisher); + + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let (closed, retry_bytes) = stage_one_zero(&publisher, &store)?; + let retry_segment = AdmittedSegment::decode(&retry_bytes, maximum_segment_policy())?; + let retry_segments = [retry_segment]; + let error = require_error( + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::one(closed, &retry_segments[0])?, + &catalog, + &retry_segments, + ), + "already-published retry ignored a recreated segment stage", + )?; + + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("already-published stage returned the wrong error".into()); + }; + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert!(matches!( + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(FilesystemCatalogPublicationError::SegmentRecoveryRequired) + )); + assert!(store.staging().join("current.seg").exists()); + drop(publisher); + store.remove() +} diff --git a/tests/catalog_publication/recording_storage.rs b/tests/catalog_publication/recording_storage.rs index cbd613b..3132083 100644 --- a/tests/catalog_publication/recording_storage.rs +++ b/tests/catalog_publication/recording_storage.rs @@ -5,7 +5,7 @@ use std::io; use keep::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationStorage, - CatalogSnapshot, ChecksummedCatalog, + CatalogSnapshot, ChecksummedCatalog, SegmentPublication, }; /// Exact complete publication order when one segment stage is present. @@ -89,6 +89,7 @@ impl CatalogPublicationStorage for RecordingStorage { &mut self, _expected: CatalogPublicationExpectation, _candidate: &CatalogSnapshot<'_, '_, '_>, + _segment: &SegmentPublication<'_, '_>, ) -> io::Result { self.record(CatalogPublicationPhase::VerifyCurrent)?; Ok(self.readiness) From 45b4e9fe76176e0e9ba467d2a839a74fa1a5dac3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:35:35 -0700 Subject: [PATCH 24/31] Fix fuzz dependency policy configuration --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 3 +++ deny.toml | 6 +++++ docs/dependencies/libfuzzer-sys-0.4.13.md | 7 +++--- .../serde-and-serde-json-1.0.229-1.0.151.md | 10 ++++---- xtask/tests/dependency_policy_contract.rs | 24 +++++++++++++++++++ 6 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 xtask/tests/dependency_policy_contract.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ede3834..d4602df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: run: cargo deny check - name: Check fuzz dependency policy - run: cargo deny --manifest-path fuzz/Cargo.toml check --config deny.toml + run: cargo deny --manifest-path fuzz/Cargo.toml check --config ../deny.toml - name: Check security advisories run: cargo audit diff --git a/CHANGELOG.md b/CHANGELOG.md index bb2293d..3e4371e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ after its public API and format compatibility policies are established. ### Changed +- The fuzz-workspace dependency gate now loads the reviewed repository + `deny.toml` explicitly and admits non-Apache licenses only through exact + package/version exceptions. - Documentation corpus selection, pinned tool admission, Markdown and fragment checks, workflow linting, Dependabot coverage, and Node lock-graph policy now run through bounded Rust `xtask` code; CI and `cargo xtask verify` use that diff --git a/deny.toml b/deny.toml index 14021c2..2432cdc 100644 --- a/deny.toml +++ b/deny.toml @@ -14,8 +14,14 @@ allow = [ exceptions = [ # `arrayref` is a required transitive dependency of locked BLAKE3 1.8.5. { allow = ["BSD-2-Clause"], crate = "arrayref@0.3.9" }, + # `libfuzzer-sys` is admitted only by the separate fuzz workspace. + { allow = ["Apache-2.0", "MIT", "NCSA"], crate = "libfuzzer-sys@0.4.13" }, + # `memchr` is activated by fuzz-only repository JSON validation. + { allow = ["MIT"], crate = "memchr@2.8.3" }, # `winx` is a required transitive dependency of locked `cap-primitives` 4.0.2. { allow = ["Apache-2.0 WITH LLVM-exception"], crate = "winx@0.36.4" }, + # `zmij` is activated by fuzz-only repository JSON validation. + { allow = ["MIT"], crate = "zmij@1.0.23" }, ] [sources] diff --git a/docs/dependencies/libfuzzer-sys-0.4.13.md b/docs/dependencies/libfuzzer-sys-0.4.13.md index e915dde..993b7ec 100644 --- a/docs/dependencies/libfuzzer-sys-0.4.13.md +++ b/docs/dependencies/libfuzzer-sys-0.4.13.md @@ -31,9 +31,10 @@ the fuzz executable workspace. Keep-owned production crates remain `unsafe_code = "forbid"`, and fuzz inputs cross into Keep exclusively through safe public byte-slice and string APIs. -The package is licensed as `(MIT OR Apache-2.0) AND NCSA`; the fuzz-specific -dependency policy contains an exact package/version exception. No global NCSA -or MIT license allowance follows from this admission. +The package is licensed as `(MIT OR Apache-2.0) AND NCSA`; the repository +dependency policy contains an exact package/version exception exercised +against the separate fuzz graph. No global NCSA or MIT license allowance +follows from this admission. ## Version, features, and transitive graph diff --git a/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md index d5286d0..6de6db3 100644 --- a/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md +++ b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md @@ -53,11 +53,11 @@ workspace lockfile. Their manifests declare minimum supported Rust versions below Keep's pinned toolchain. -The fuzz-workspace license policy retains Apache-2.0 as its default allowlist -and grants exact-version MIT exceptions to `memchr` 2.8.3 and `zmij` 1.0.23. -Those exceptions admit only the reviewed transitive graph named above; a -resolved version change remains a policy failure until this record and the -exception are reviewed together. +The repository license policy retains Apache-2.0 as its default allowlist and +grants exact-version MIT exceptions to `memchr` 2.8.3 and `zmij` 1.0.23 when +it checks the separate fuzz graph. Those exceptions admit only the reviewed +transitive graph named above; a resolved version change remains a policy +failure until this record and the exception are reviewed together. Keep-owned code invokes only safe APIs. The parser and its transitive dependencies may contain implementation details outside Keep's `unsafe_code` diff --git a/xtask/tests/dependency_policy_contract.rs b/xtask/tests/dependency_policy_contract.rs new file mode 100644 index 0000000..68dbb17 --- /dev/null +++ b/xtask/tests/dependency_policy_contract.rs @@ -0,0 +1,24 @@ +//! Repository dependency-policy configuration regression evidence. + +const CI_WORKFLOW: &str = include_str!("../../.github/workflows/ci.yml"); +const POLICY: &str = include_str!("../../deny.toml"); + +#[test] +fn fuzz_dependency_gate_uses_the_reviewed_repository_policy() { + assert!( + CI_WORKFLOW + .contains("cargo deny --manifest-path fuzz/Cargo.toml check --config ../deny.toml") + ); +} + +#[test] +fn fuzz_only_license_exceptions_are_package_and_version_scoped() { + for exception in [ + r#"crate = "libfuzzer-sys@0.4.13""#, + r#"crate = "memchr@2.8.3""#, + r#"crate = "winx@0.36.4""#, + r#"crate = "zmij@1.0.23""#, + ] { + assert!(POLICY.contains(exception), "missing {exception}"); + } +} From c3cc0bd4f636b3a8b3d4c7c88a83493ea2171db3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:36:34 -0700 Subject: [PATCH 25/31] Update fuzz lock for capability dependencies --- fuzz/Cargo.lock | 318 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 318 insertions(+) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 7d8e423..7a86f0f 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "arbitrary" version = "1.4.2" @@ -20,6 +26,12 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "blake3" version = "1.8.5" @@ -34,6 +46,48 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "cap-fs-ext" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78e5a3368ae89b7cb68186411452b4b9fac8b41be9c19bf3f47c2d2c8e36e6b" +dependencies = [ + "cap-primitives", + "cap-std", + "io-lifetimes 3.0.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "cap-primitives" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdadbd7c002d3a484b35243669abdae85a0ebaded5a61117169dc3400f9a7ff0" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7281235d6e96d3544ca18bba9049be92f4190f8d923e3caef1b5f66cfa752608" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + [[package]] name = "cc" version = "1.3.0" @@ -67,12 +121,33 @@ dependencies = [ "libc", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -84,6 +159,34 @@ dependencies = [ "r-efi", ] +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.60.2", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "itoa" version = "1.0.18" @@ -105,6 +208,8 @@ name = "keep" version = "0.0.0" dependencies = [ "blake3", + "cap-fs-ext", + "cap-std", ] [[package]] @@ -132,12 +237,30 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -162,6 +285,29 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "serde" version = "1.0.229" @@ -227,6 +373,178 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags", + "windows-sys 0.59.0", +] + [[package]] name = "xtask" version = "0.0.0" From a8e3af103e4726f4357a88246a0a511137f956f2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 02:42:59 -0700 Subject: [PATCH 26/31] Fuzz catalog format boundaries --- CHANGELOG.md | 4 +- docs/formats/segment-store-v1/requirements.md | 6 ++- fuzz/Cargo.toml | 7 +++ fuzz/README.md | 5 ++ fuzz/fuzz_targets/catalog_format.rs | 33 +++++++++++++ xtask/src/fuzz_campaign/target/tests.rs | 1 + xtask/src/fuzz_seed_corpus.rs | 2 + xtask/src/fuzz_seed_corpus/catalog_seeds.rs | 49 +++++++++++++++++++ .../fuzz_seed_corpus/tests/materialization.rs | 21 +++++++- .../tests/segment_store_protocol_contract.rs | 2 + .../parser_fuzz_laws.rs | 26 ++++++++++ 11 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 fuzz/fuzz_targets/catalog_format.rs create mode 100644 xtask/src/fuzz_seed_corpus/catalog_seeds.rs create mode 100644 xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e4371e..2720061 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -191,8 +191,8 @@ after its public API and format compatibility policies are established. exact logical-record-to-segment admission with one bounded physical lookup plan, one scan per referenced segment, and refusal of every unreferenced caller-supplied segment during construction or admission; deterministic - successor proofs; immutable reader snapshots; and `BTreeMap` transition-model evidence for - `keep.segment-store/v1`. + successor proofs; immutable reader snapshots; seeded parser fuzzing; and + `BTreeMap` transition-model evidence for `keep.segment-store/v1`. - Blocking `FilesystemCatalogPublisher` publication under a persistent kernel-managed writer lock, with pinned directory capabilities, no-replacement immutable-pool links, complete post-link verification, diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 9a52351..15d6bd1 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -76,6 +76,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | +| `KEEP-CATALOG-011` | Every public catalog and publication-head parser boundary is fuzzed from canonical deterministic seeds | Canonical generation-1, generation-2, and bundle artifacts | `fuzz/fuzz_targets/catalog_format.rs`, `xtask/src/fuzz_seed_corpus/catalog_seeds.rs` | Implemented in #16 | @@ -130,8 +131,9 @@ tables and formulas. The issue #15 segment implementation matches the frozen segment corpus and adds parser fuzzing and corruption evidence. Issue #16 matches the catalog and publication-head corpus, executes the documented publication order through a real filesystem adapter, reconstructs exact -immutable restart snapshots, and adds deterministic transition-model -evidence. Crash-injection and explicit recovery remain owned by issue #17. +immutable restart snapshots, and adds deterministic transition-model and +seeded parser-fuzz evidence. Crash-injection and explicit recovery remain +owned by issue #17. The format-local tradeoffs are recorded in the [colocated rationale](rationale.md). diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index bb29289..5ab41b7 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -89,6 +89,13 @@ test = false doc = false bench = false +[[bin]] +name = "catalog_format" +path = "fuzz_targets/catalog_format.rs" +test = false +doc = false +bench = false + [[bin]] name = "fast_cdc" path = "fuzz_targets/fast_cdc.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 4cb71eb..0bcff69 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -60,6 +60,11 @@ from the reviewed hexadecimal fixtures. The target feeds arbitrary bytes through the bounded decoder and, for every admitted record, requires exact decode-encode byte equality plus successful expected-`LayoutId` admission. +The `catalog_format` seeds select the public catalog and publication-head +decoders. Canonical generation-1, generation-2, and two-record bundle +artifacts keep mutations inside framing, ordering, coordinate, checksum, and +digest validation; every admitted value must retain its exact input bytes. + The `segment_format` seeds select the public segment-header, record-header, complete-record, seal, and complete-segment boundaries. Canonical empty, one-record, and bundled segments keep mutations inside the nested parsers; diff --git a/fuzz/fuzz_targets/catalog_format.rs b/fuzz/fuzz_targets/catalog_format.rs new file mode 100644 index 0000000..348e2f9 --- /dev/null +++ b/fuzz/fuzz_targets/catalog_format.rs @@ -0,0 +1,33 @@ +#![no_main] + +//! This target owns canonical catalog and publication-head parser fuzzing. + +use keep::{ChecksummedCatalog, ChecksummedPublicationHead}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|bytes: &[u8]| { + let Some((&selector, input)) = bytes.split_first() else { + return; + }; + if selector == 0 { + catalog(input); + } else { + publication_head(input); + } +}); + +fn catalog(input: &[u8]) { + if let Ok(catalog) = ChecksummedCatalog::decode(input) { + assert_eq!(catalog.encoded(), input); + assert_eq!( + u64::try_from(catalog.encoded().len()), + Ok(catalog.length().get()) + ); + } +} + +fn publication_head(input: &[u8]) { + if let Ok(head) = ChecksummedPublicationHead::decode(input) { + assert_eq!(head.encoded(), input); + } +} diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index 03518e1..82e88b3 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -26,6 +26,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "blob_hasher", "blob_id_binary", "blob_id_text", + "catalog_format", "fast_cdc", "golden_protocol", "layout_record", diff --git a/xtask/src/fuzz_seed_corpus.rs b/xtask/src/fuzz_seed_corpus.rs index cb97c28..61c76c0 100644 --- a/xtask/src/fuzz_seed_corpus.rs +++ b/xtask/src/fuzz_seed_corpus.rs @@ -1,5 +1,6 @@ //! This module owns deterministic fuzz seed recipes and materialization. +mod catalog_seeds; mod cdc_seeds; mod filesystem; mod identity_seeds; @@ -64,6 +65,7 @@ impl Seed { pub(super) fn prepare(repository_root: &Path) -> Result<(), FuzzSeedError> { let files = RepositoryFiles::open(repository_root)?; let mut seeds = identity_seeds::seeds(&files)?; + seeds.extend(catalog_seeds::seeds(&files)?); seeds.extend(cdc_seeds::seeds()?); seeds.extend(golden_protocol_seeds_from(&files)?); seeds.extend(layout_seeds::seeds(&files)?); diff --git a/xtask/src/fuzz_seed_corpus/catalog_seeds.rs b/xtask/src/fuzz_seed_corpus/catalog_seeds.rs new file mode 100644 index 0000000..6f4fadc --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/catalog_seeds.rs @@ -0,0 +1,49 @@ +//! This module owns canonical catalog and publication-head fuzz seeds. + +use std::path::Path; + +use super::filesystem::RepositoryFiles; +use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; +use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; + +const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v1"; + +pub(super) const FIXTURES: [(u8, &str); 6] = [ + (0, "one-zero-catalog.hex"), + (0, "one-zero-catalog-generation-two.hex"), + (0, "one-zero-bundle-catalog.hex"), + (1, "one-zero-head.hex"), + (1, "one-zero-head-generation-two.hex"), + (1, "one-zero-bundle-head.hex"), +]; + +pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { + let mut seeds = Vec::new(); + for (selector, fixture) in FIXTURES { + let name = fixture + .strip_suffix(".hex") + .ok_or_else(|| FuzzSeedError::violation("catalog fixture lacks .hex suffix"))?; + let encoded = fixture_bytes(files, fixture)?; + seeds.push(Seed::new( + "catalog_format", + name, + prefixed(selector, &encoded)?, + )?); + } + Ok(seeds) +} + +fn fixture_bytes(files: &RepositoryFiles, fixture: &'static str) -> Result, FuzzSeedError> { + let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); + let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; + let lines = framed_lines(&transport, MAX_SEED_BYTES) + .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; + let [encoded] = lines.as_slice() else { + return Err(FuzzSeedError::violation(format!( + "{fixture} must contain exactly one hexadecimal line" + ))); + }; + decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { + FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) + }) +} diff --git a/xtask/src/fuzz_seed_corpus/tests/materialization.rs b/xtask/src/fuzz_seed_corpus/tests/materialization.rs index eb813f2..856ccf5 100644 --- a/xtask/src/fuzz_seed_corpus/tests/materialization.rs +++ b/xtask/src/fuzz_seed_corpus/tests/materialization.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::path::Path; -use super::super::{FuzzSeedError, layout_seeds, prepare, segment_seeds}; +use super::super::{FuzzSeedError, catalog_seeds, layout_seeds, prepare, segment_seeds}; use crate::test_directory::TestDirectory; const TABLES: [&str; 5] = [ @@ -38,11 +38,13 @@ fn seed_preparation_materializes_the_complete_deterministic_set() } copy_layout_fixtures(source_root, root)?; copy_segment_fixtures(source_root, root)?; + copy_catalog_fixtures(source_root, root)?; prepare(root)?; let corpus = root.join("fuzz/corpus"); let first = seed_contents(&corpus)?; - assert_eq!(first.len(), 34); + assert_eq!(first.len(), 40); + assert_eq!(target_seed_count(&first, "catalog_format/"), 6); assert_eq!(target_seed_count(&first, "golden_protocol/"), 9); assert_eq!(target_seed_count(&first, "layout_record/"), 4); assert_eq!(target_seed_count(&first, "segment_format/"), 8); @@ -95,6 +97,21 @@ fn copy_segment_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeed Ok(()) } +fn copy_catalog_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { + use std::fs; + + let catalog_directory = root.join("conformance/segment-store/v1"); + for (_selector, fixture) in catalog_seeds::FIXTURES { + let source_path = source_root + .join("conformance/segment-store/v1") + .join(fixture); + let destination = catalog_directory.join(fixture); + fs::copy(&source_path, &destination) + .map_err(|source| FuzzSeedError::io("copy test catalog", &destination, source))?; + } + Ok(()) +} + fn target_seed_count(contents: &BTreeMap>, prefix: &str) -> usize { contents .keys() diff --git a/xtask/tests/segment_store_protocol_contract.rs b/xtask/tests/segment_store_protocol_contract.rs index bc1cb95..f9334de 100644 --- a/xtask/tests/segment_store_protocol_contract.rs +++ b/xtask/tests/segment_store_protocol_contract.rs @@ -8,6 +8,8 @@ use std::path::Path; mod documentation_laws; #[path = "segment_store_protocol_contract/fixture_oracle.rs"] mod fixture_oracle; +#[path = "segment_store_protocol_contract/parser_fuzz_laws.rs"] +mod parser_fuzz_laws; #[path = "segment_store_protocol_contract/publication_laws.rs"] mod publication_laws; #[path = "segment_store_protocol_contract/recovery_laws.rs"] diff --git a/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs new file mode 100644 index 0000000..10602b9 --- /dev/null +++ b/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs @@ -0,0 +1,26 @@ +//! Fuzz-evidence laws for durable catalog parser boundaries. + +use std::error::Error; +use std::path::Path; + +const FUZZ_MANIFEST: &str = include_str!("../../../fuzz/Cargo.toml"); +const FUZZ_GUIDE: &str = include_str!("../../../fuzz/README.md"); +const REQUIREMENTS: &str = include_str!("../../../docs/formats/segment-store-v1/requirements.md"); + +#[test] +fn catalog_and_head_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box> { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest must have a repository parent")?; + + assert!( + repository_root + .join("fuzz/fuzz_targets/catalog_format.rs") + .is_file() + ); + assert!(FUZZ_MANIFEST.contains("name = \"catalog_format\"")); + assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/catalog_format.rs\"")); + assert!(FUZZ_GUIDE.contains("The `catalog_format` seeds")); + assert!(REQUIREMENTS.contains("`KEEP-CATALOG-011`")); + Ok(()) +} From 7e33c16d9443404559ad2922d54607791fd72393 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 03:03:11 -0700 Subject: [PATCH 27/31] Fix Linux catalog directory synchronization --- CHANGELOG.md | 3 ++ src/adapters/filesystem_catalog_publisher.rs | 17 ++++--- src/adapters/mod.rs | 1 + src/adapters/sync_capable_directory.rs | 25 ++++++++++ tests/catalog_filesystem_publication.rs | 2 + .../directory_laws.rs | 47 +++++++++++++++++++ 6 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 src/adapters/sync_capable_directory.rs create mode 100644 tests/catalog_filesystem_publication/directory_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2720061..4a7baf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ after its public API and format compatibility policies are established. ### Changed +- Filesystem catalog publishers now retain no-follow, read-capable directory + handles so required durability synchronization works on Linux instead of + failing on `O_PATH` descriptors. - The fuzz-workspace dependency gate now loads the reviewed repository `deny.toml` explicitly and admits non-Apache licenses only through exact package/version exceptions. diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 071fb48..2b11a52 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -2,11 +2,11 @@ use std::io; -use cap_fs_ext::DirExt; use cap_std::fs::{Dir, File}; use super::{ CatalogRestartPolicy, FilesystemSegmentStage, FilesystemWriterLock, SegmentStageCreateError, + sync_capable_directory, }; pub(super) const CURRENT_SEGMENT: &str = "current.seg"; @@ -39,13 +39,16 @@ impl FilesystemCatalogPublisher { /// /// # Errors /// - /// Returns the exact root-clone or no-follow directory-open failure. A - /// failure drops `lock` and therefore releases writer authority. + /// Returns the exact root-clone, no-follow directory-open, or + /// directory-inspection failure. A namespace entry that is not a directory + /// returns [`io::ErrorKind::NotADirectory`]. A failure drops `lock` and + /// therefore releases writer authority. pub fn open(lock: FilesystemWriterLock, policy: CatalogRestartPolicy) -> io::Result { - let root = lock.clone_directory()?; - let staging = root.open_dir_nofollow("staging")?; - let segments = root.open_dir_nofollow("segments")?; - let catalogs = root.open_dir_nofollow("catalogs")?; + let pinned_root = lock.clone_directory()?; + let root = sync_capable_directory::open(&pinned_root, ".")?; + let staging = sync_capable_directory::open(&root, "staging")?; + let segments = sync_capable_directory::open(&root, "segments")?; + let catalogs = sync_capable_directory::open(&root, "catalogs")?; Ok(Self { root, staging, diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index e54ffed..291d66a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -155,6 +155,7 @@ mod segment_write_phase; mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; +mod sync_capable_directory; mod writer_lock_acquire_error; mod writer_lock_acquire_phase; diff --git a/src/adapters/sync_capable_directory.rs b/src/adapters/sync_capable_directory.rs new file mode 100644 index 0000000..0951a11 --- /dev/null +++ b/src/adapters/sync_capable_directory.rs @@ -0,0 +1,25 @@ +//! This module owns sync-capable capability directory admission. + +use std::io; + +use cap_fs_ext::{ + FollowSymlinks, OpenOptionsFollowExt, OpenOptionsMaybeDirExt, OpenOptionsSyncExt, +}; +use cap_std::fs::{Dir, OpenOptions}; + +pub(super) fn open(parent: &Dir, name: &str) -> io::Result { + let mut options = OpenOptions::new(); + options + .read(true) + .maybe_dir(true) + .follow(FollowSymlinks::No) + .nonblock(true); + let file = parent.open_with(name, &options)?; + if !file.metadata()?.is_dir() { + return Err(io::Error::new( + io::ErrorKind::NotADirectory, + "sync-capable target is not a directory", + )); + } + Ok(Dir::from_std_file(file.into_std())) +} diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index e3ddc4d..037ca0d 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -1,5 +1,7 @@ //! Filesystem-backed catalog publication laws. +#[path = "catalog_filesystem_publication/directory_laws.rs"] +mod directory_laws; #[path = "catalog_filesystem_publication/refusal_laws.rs"] mod refusal_laws; #[path = "segment_filesystem_stage/sandbox.rs"] diff --git a/tests/catalog_filesystem_publication/directory_laws.rs b/tests/catalog_filesystem_publication/directory_laws.rs new file mode 100644 index 0000000..4fbea61 --- /dev/null +++ b/tests/catalog_filesystem_publication/directory_laws.rs @@ -0,0 +1,47 @@ +use std::error::Error; +use std::fs; +use std::io; + +use keep::{FilesystemCatalogPublisher, FilesystemWriterLock}; + +use super::{StoreFixture, restart_policy}; + +#[test] +fn publisher_refuses_a_non_directory_protocol_namespace() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-nondirectory")?; + fs::remove_dir(store.staging())?; + fs::write(store.staging(), [])?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + + let Err(error) = FilesystemCatalogPublisher::open(lock, restart_policy()?) else { + return Err("publisher admitted a non-directory namespace".into()); + }; + + assert_eq!(error.kind(), io::ErrorKind::NotADirectory); + store.remove() +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn publisher_never_follows_a_symbolic_protocol_namespace() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + #[cfg(target_os = "linux")] + const SYMBOLIC_LINK_LOOP_ERROR: i32 = 40; + #[cfg(target_os = "macos")] + const SYMBOLIC_LINK_LOOP_ERROR: i32 = 62; + + let store = StoreFixture::create("catalog-filesystem-directory-link")?; + fs::remove_dir(store.staging())?; + let target = store.path().join("replacement-staging"); + fs::create_dir(&target)?; + symlink(&target, store.staging())?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + + let Err(error) = FilesystemCatalogPublisher::open(lock, restart_policy()?) else { + return Err("publisher followed a symbolic namespace".into()); + }; + + assert_eq!(error.raw_os_error(), Some(SYMBOLIC_LINK_LOOP_ERROR)); + store.remove() +} From a1c5acbd023bbeb3d43ae78f479d5f8921f6d137 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 03:33:13 -0700 Subject: [PATCH 28/31] Bind segment stages to publisher authority --- CHANGELOG.md | 12 ++- README.md | 8 +- docs/formats/segment-store-v1/publication.md | 15 +-- src/adapters/filesystem_catalog_current.rs | 14 +++ .../filesystem_catalog_publication_error.rs | 5 + src/adapters/filesystem_catalog_publisher.rs | 39 ++++++- .../filesystem_publisher_authority.rs | 25 +++++ src/adapters/filesystem_segment_stage.rs | 16 ++- src/adapters/mod.rs | 1 + src/adapters/sealed_segment.rs | 23 ++-- src/adapters/segment_publication.rs | 36 ++++++- src/adapters/segment_publication_error.rs | 5 + tests/catalog_filesystem_publication.rs | 33 +++--- .../authority_laws.rs | 102 ++++++++++++++++++ .../refusal_laws.rs | 27 ++--- 15 files changed, 309 insertions(+), 52 deletions(-) create mode 100644 src/adapters/filesystem_publisher_authority.rs create mode 100644 tests/catalog_filesystem_publication/authority_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7baf7..c77899f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Filesystem segment selection now consumes sealed stages through the publisher + that created them. Process-local publisher authority prevents an unrelated + metadata-equivalent `ClosedSegment` from authorizing retained + `staging/current.seg` bytes. - Filesystem catalog publishers now retain no-follow, read-capable directory handles so required durability synchronization works on Linux instead of failing on `O_PATH` descriptors. @@ -201,9 +205,11 @@ after its public API and format compatibility policies are established. no-replacement immutable-pool links, complete post-link verification, explicit file and directory synchronization, transitive `head.next` verification, atomic `HEAD` replacement, and stale or recovery-required - refusal before mutation. New segment publication consumes a handle-free - `ClosedSegment` proof before any immutable-pool link, and publisher teardown - closes every retained writable handle before releasing writer authority. + refusal before mutation. New filesystem segment publication consumes the + sealed stage through its creating publisher, checks process-local publisher + authority, and closes the writable handle before any immutable-pool link; + publisher teardown closes every retained writable handle before releasing + writer authority. Retry of an already-current complete candidate re-synchronizes the root and returns an explicit `CatalogPublicationOutcome::AlreadyPublished` receipt without repeating publication mutations. Retained `head.next` or diff --git a/README.md b/README.md index d262e87..10d4428 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,11 @@ root, staging, segment-pool, and catalog-pool capabilities for the complete blocking publication. It reopens and verifies synchronized stages, uses no-replacement immutable-pool links, synchronizes every required file and directory, verifies the complete `head.next` view, atomically replaces `HEAD`, -and returns a receipt only after root synchronization. New segment publication -requires `SealedSegment::close` to consume the writable stage and bind its -handle-free `ClosedSegment` receipt to exact admitted bytes. +and returns a receipt only after root synchronization. New filesystem segment +publication requires `FilesystemCatalogPublisher::select_segment` to consume +the sealed writable stage, prove that this publisher created it, and bind its +synchronized metadata to exact admitted bytes. A storage-agnostic +`ClosedSegment` receipt alone cannot authorize a retained filesystem stage. `FilesystemCatalogSnapshot` follows only the exact checksummed head, catalog, and segment coordinates and retains caller-bounded immutable bytes for pinned logical reads. diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index 49977ae..df69ce0 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -134,12 +134,15 @@ reopened and compared against the preflighted canonical artifact before the protocol advances. Publisher teardown closes retained writable handles and pinned directory capabilities before releasing the writer lock. -Publishing a new segment additionally requires a checked -`SegmentPublication::one` selection. The caller must first consume -`SealedSegment::close`, which drops Keep's owned writable stage before -returning a handle-free `ClosedSegment` receipt. Selection binds that receipt's -record count, byte length, and digest to the exact `AdmittedSegment` bytes. -Catalog publication cannot select an unrelated or still-open sealed stage. +Publishing a new filesystem segment additionally requires +`FilesystemCatalogPublisher::select_segment`. The method consumes the +`SealedSegment`, drops Keep's owned writable stage, proves that this publisher +created the stage, and binds its record count, byte length, and digest to the +exact `AdmittedSegment` bytes. A storage-agnostic +`SegmentPublication::one` selection remains available to non-filesystem +adapters, but its handle-free `ClosedSegment` receipt cannot authorize +`staging/current.seg`. Catalog publication cannot select an unrelated or +still-open sealed stage. `FilesystemCatalogPublisher::create_segment_stage` is the only public filesystem-stage constructor. Its returned lifetime keeps the acquired writer authority borrowed while `current.seg` remains writable. diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs index d1e238e..682abe2 100644 --- a/src/adapters/filesystem_catalog_current.rs +++ b/src/adapters/filesystem_catalog_current.rs @@ -15,6 +15,7 @@ pub(super) fn verify( candidate: &CatalogSnapshot<'_, '_, '_>, segment: &SegmentPublication<'_, '_>, ) -> io::Result { + require_segment_authority(publisher, segment)?; require_no_next_head(publisher)?; require_no_catalog_stage(publisher)?; if segment.admitted().is_none() { @@ -58,6 +59,19 @@ pub(super) fn verify( Ok(readiness) } +fn require_segment_authority( + publisher: &FilesystemCatalogPublisher, + segment: &SegmentPublication<'_, '_>, +) -> io::Result<()> { + if segment.is_bound_to(&publisher.authority) { + Ok(()) + } else { + Err(filesystem_catalog_artifact::invalid_data( + FilesystemCatalogPublicationError::SegmentAuthorityRequired, + )) + } +} + fn require_no_next_head(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { match publisher .root diff --git a/src/adapters/filesystem_catalog_publication_error.rs b/src/adapters/filesystem_catalog_publication_error.rs index ec54739..d8c7507 100644 --- a/src/adapters/filesystem_catalog_publication_error.rs +++ b/src/adapters/filesystem_catalog_publication_error.rs @@ -9,6 +9,8 @@ use crate::{CatalogDigest, CatalogGeneration}; /// Filesystem state disagreed with a preflighted publication invariant. #[derive(Debug)] pub enum FilesystemCatalogPublicationError { + /// The selected segment has no authority from this publisher. + SegmentAuthorityRequired, /// The current publication coordinate was stale or unexpectedly present. CurrentState { /// Generation required by the caller, absent for initialization. @@ -41,6 +43,9 @@ pub enum FilesystemCatalogPublicationError { impl fmt::Display for FilesystemCatalogPublicationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::SegmentAuthorityRequired => { + formatter.write_str("selected segment lacks publisher stage authority") + } Self::CurrentState { .. } => { formatter.write_str("current catalog publication state is stale") } diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 2b11a52..9d3976d 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -4,9 +4,11 @@ use std::io; use cap_std::fs::{Dir, File}; +use super::filesystem_publisher_authority::FilesystemPublisherAuthority; use super::{ - CatalogRestartPolicy, FilesystemSegmentStage, FilesystemWriterLock, SegmentStageCreateError, - sync_capable_directory, + AdmittedSegment, CatalogRestartPolicy, ClosedSegment, FilesystemSegmentStage, + FilesystemWriterLock, SealedSegment, SegmentPublication, SegmentPublicationError, + SegmentStageCreateError, sync_capable_directory, }; pub(super) const CURRENT_SEGMENT: &str = "current.seg"; @@ -27,6 +29,7 @@ pub struct FilesystemCatalogPublisher { pub(super) segments: Dir, pub(super) catalogs: Dir, pub(super) policy: CatalogRestartPolicy, + pub(super) authority: FilesystemPublisherAuthority, pub(super) catalog_stage: Option, pub(super) head_stage: Option, // Fields drop in declaration order. Writer authority must outlive every @@ -42,7 +45,8 @@ impl FilesystemCatalogPublisher { /// Returns the exact root-clone, no-follow directory-open, or /// directory-inspection failure. A namespace entry that is not a directory /// returns [`io::ErrorKind::NotADirectory`]. A failure drops `lock` and - /// therefore releases writer authority. + /// therefore releases writer authority. Success allocates one ephemeral + /// authority token that binds later stage selection to this publisher. pub fn open(lock: FilesystemWriterLock, policy: CatalogRestartPolicy) -> io::Result { let pinned_root = lock.clone_directory()?; let root = sync_capable_directory::open(&pinned_root, ".")?; @@ -55,6 +59,7 @@ impl FilesystemCatalogPublisher { segments, catalogs, policy, + authority: FilesystemPublisherAuthority::new(), catalog_stage: None, head_stage: None, _lock: lock, @@ -76,4 +81,32 @@ impl FilesystemCatalogPublisher { ) -> Result, SegmentStageCreateError> { FilesystemSegmentStage::create(self) } + + /// Closes and selects one synchronized stage created by this publisher. + /// + /// The returned selection remains bound to this exact publisher instance + /// and cannot authorize a metadata-equivalent retained stage owned by + /// another publisher or storage implementation. + /// + /// # Errors + /// + /// Returns [`SegmentPublicationError::PublisherAuthority`] when `sealed` + /// was created by another publisher. Other variants preserve exact + /// closed-stage to admitted-segment disagreements. + pub fn select_segment<'selection, 'records>( + &self, + sealed: SealedSegment>, + admitted: &'selection AdmittedSegment<'records>, + ) -> Result, SegmentPublicationError> { + let (stage, record_count, segment_length, digest) = sealed.into_parts(); + let authority = stage.close(); + if !self.authority.matches(&authority) { + return Err(SegmentPublicationError::PublisherAuthority); + } + SegmentPublication::one_bound( + ClosedSegment::admitted(record_count, segment_length, digest), + admitted, + authority, + ) + } } diff --git a/src/adapters/filesystem_publisher_authority.rs b/src/adapters/filesystem_publisher_authority.rs new file mode 100644 index 0000000..095075e --- /dev/null +++ b/src/adapters/filesystem_publisher_authority.rs @@ -0,0 +1,25 @@ +//! This module owns one publisher's ephemeral stage authority. +//! +//! One allocation supplies stable, non-forgeable process-local identity across +//! publisher moves. The token never enters a durable format or content identity. + +use std::sync::Arc; + +#[derive(Clone)] +pub(super) struct FilesystemPublisherAuthority { + token: Arc, +} + +struct AuthorityToken; + +impl FilesystemPublisherAuthority { + pub(super) fn new() -> Self { + Self { + token: Arc::new(AuthorityToken), + } + } + + pub(super) fn matches(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.token, &other.token) + } +} diff --git a/src/adapters/filesystem_segment_stage.rs b/src/adapters/filesystem_segment_stage.rs index bdb9159..1d54057 100644 --- a/src/adapters/filesystem_segment_stage.rs +++ b/src/adapters/filesystem_segment_stage.rs @@ -6,6 +6,7 @@ use cap_std::fs::File; use super::filesystem_catalog_artifact; use super::filesystem_catalog_publisher::CURRENT_SEGMENT; +use super::filesystem_publisher_authority::FilesystemPublisherAuthority; use super::{FilesystemCatalogPublisher, SegmentStage, SegmentStageCreateError}; /// An exclusively created empty `current.seg` staging file under writer authority. @@ -18,9 +19,11 @@ use super::{FilesystemCatalogPublisher, SegmentStage, SegmentStageCreateError}; /// The lifetime keeps the locked publisher borrowed until the writable file is /// closed. This type does not synchronize the staging-directory entry, publish /// the file, or establish that the surrounding filesystem satisfies Keep's -/// complete platform contract. +/// complete platform contract. It retains this publisher's private authority +/// token so the sealed stage cannot be substituted by metadata alone. pub struct FilesystemSegmentStage<'publisher> { file: File, + authority: FilesystemPublisherAuthority, _publisher: &'publisher FilesystemCatalogPublisher, } @@ -33,9 +36,20 @@ impl<'publisher> FilesystemSegmentStage<'publisher> { .map_err(|source| SegmentStageCreateError::Create { source })?; Ok(Self { file, + authority: publisher.authority.clone(), _publisher: publisher, }) } + + pub(super) fn close(self) -> FilesystemPublisherAuthority { + let Self { + file, + authority, + _publisher: _, + } = self; + drop(file); + authority + } } impl Write for FilesystemSegmentStage<'_> { diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 291d66a..1d0c7a0 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -72,6 +72,7 @@ mod filesystem_catalog_publisher; mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; mod filesystem_catalog_storage; +mod filesystem_publisher_authority; mod filesystem_segment_stage; mod filesystem_writer_lock; mod framed_blake3; diff --git a/src/adapters/sealed_segment.rs b/src/adapters/sealed_segment.rs index 7dfdd23..de68d20 100644 --- a/src/adapters/sealed_segment.rs +++ b/src/adapters/sealed_segment.rs @@ -22,18 +22,15 @@ impl SealedSegment where S: SegmentStage, { - /// Closes the owned writable stage and returns publication-safe metadata. + /// Closes the owned writable stage and returns storage-agnostic metadata. /// /// The stage has already completed both required flush-and-sync /// transitions. This consuming operation drops the only stage value Keep - /// owns before returning a handle-free [`ClosedSegment`] receipt. + /// owns before returning a handle-free [`ClosedSegment`] receipt. A + /// filesystem publisher requires its own authority-bound selection method; + /// this generic receipt cannot authorize a retained filesystem stage. pub fn close(self) -> ClosedSegment { - let Self { - _stage: stage, - record_count, - segment_length, - digest, - } = self; + let (stage, record_count, segment_length, digest) = self.into_parts(); drop(stage); ClosedSegment::admitted(record_count, segment_length, digest) } @@ -56,6 +53,16 @@ where self.digest } + pub(super) fn into_parts(self) -> (S, u32, u64, SegmentDigest) { + let Self { + _stage: stage, + record_count, + segment_length, + digest, + } = self; + (stage, record_count, segment_length, digest) + } + pub(super) const fn admitted( stage: S, record_count: u32, diff --git a/src/adapters/segment_publication.rs b/src/adapters/segment_publication.rs index 86d6005..3198754 100644 --- a/src/adapters/segment_publication.rs +++ b/src/adapters/segment_publication.rs @@ -1,11 +1,14 @@ //! This module owns optional closed-segment publication selection. +use super::filesystem_publisher_authority::FilesystemPublisherAuthority; use super::{AdmittedSegment, ClosedSegment, SegmentPublicationError}; /// Segment-pool work required before publishing one catalog generation. /// /// The selected form can be created only by consuming a handle-free /// [`ClosedSegment`] receipt and binding it to the exact admitted stage bytes. +/// Storage adapters may require additional private provenance before they +/// accept that selection. #[must_use] pub struct SegmentPublication<'selection, 'records> { selected: Option>, @@ -14,6 +17,7 @@ pub struct SegmentPublication<'selection, 'records> { struct SelectedSegment<'selection, 'records> { _closed: ClosedSegment, admitted: &'selection AdmittedSegment<'records>, + authority: Option, } impl<'selection, 'records> SegmentPublication<'selection, 'records> { @@ -22,7 +26,11 @@ impl<'selection, 'records> SegmentPublication<'selection, 'records> { Self { selected: None } } - /// Binds one closed synchronized stage to its exact admitted bytes. + /// Binds one storage-agnostic closed stage to its exact admitted bytes. + /// + /// This selection carries no filesystem-publisher authority. Use + /// [`crate::FilesystemCatalogPublisher::select_segment`] for filesystem + /// publication. /// /// # Errors /// @@ -31,6 +39,22 @@ impl<'selection, 'records> SegmentPublication<'selection, 'records> { pub fn one( closed: ClosedSegment, admitted: &'selection AdmittedSegment<'records>, + ) -> Result { + Self::bind(closed, admitted, None) + } + + pub(super) fn one_bound( + closed: ClosedSegment, + admitted: &'selection AdmittedSegment<'records>, + authority: FilesystemPublisherAuthority, + ) -> Result { + Self::bind(closed, admitted, Some(authority)) + } + + fn bind( + closed: ClosedSegment, + admitted: &'selection AdmittedSegment<'records>, + authority: Option, ) -> Result { let observed_length = u64::try_from(admitted.encoded().len()).map_err(|_source| { SegmentPublicationError::HostLength { @@ -59,6 +83,7 @@ impl<'selection, 'records> SegmentPublication<'selection, 'records> { selected: Some(SelectedSegment { _closed: closed, admitted, + authority, }), }) } @@ -73,4 +98,13 @@ impl<'selection, 'records> SegmentPublication<'selection, 'records> { pub(super) fn into_admitted(self) -> Option<&'selection AdmittedSegment<'records>> { self.selected.map(|selected| selected.admitted) } + + pub(super) fn is_bound_to(&self, authority: &FilesystemPublisherAuthority) -> bool { + self.selected.as_ref().is_none_or(|selected| { + selected + .authority + .as_ref() + .is_some_and(|selected| selected.matches(authority)) + }) + } } diff --git a/src/adapters/segment_publication_error.rs b/src/adapters/segment_publication_error.rs index 019b065..63b7ada 100644 --- a/src/adapters/segment_publication_error.rs +++ b/src/adapters/segment_publication_error.rs @@ -8,6 +8,8 @@ use super::SegmentDigest; /// A closed stage receipt disagreed with the admitted segment selected for publication. #[derive(Debug)] pub enum SegmentPublicationError { + /// The closed filesystem stage belongs to another publisher. + PublisherAuthority, /// The admitted segment byte length cannot be represented by the protocol. HostLength { /// Host byte length that could not be represented. @@ -39,6 +41,9 @@ pub enum SegmentPublicationError { impl fmt::Display for SegmentPublicationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::PublisherAuthority => { + formatter.write_str("sealed segment belongs to another publisher") + } Self::HostLength { .. } => { formatter.write_str("admitted segment length is not representable") } diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index 037ca0d..c30a19b 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -1,5 +1,7 @@ //! Filesystem-backed catalog publication laws. +#[path = "catalog_filesystem_publication/authority_laws.rs"] +mod authority_laws; #[path = "catalog_filesystem_publication/directory_laws.rs"] mod directory_laws; #[path = "catalog_filesystem_publication/refusal_laws.rs"] @@ -15,9 +17,10 @@ use std::path::{Path, PathBuf}; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogRestartByteLimit, - CatalogRestartPolicy, ClosedSegment, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemWriterLock, LayoutEntryLimit, SegmentPublication, SegmentReadPolicy, - SegmentRecordLimit, StagedSegment, publish_catalog_generation, + CatalogRestartPolicy, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemSegmentStage, FilesystemWriterLock, LayoutEntryLimit, SealedSegment, + SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, + publish_catalog_generation, }; use sandbox::TestDirectory; use support::decode_hex; @@ -30,19 +33,19 @@ const CATALOG_DIGEST: &str = "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03 const SEGMENT_DIGEST: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc"; const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; -type StagedFixture = (ClosedSegment, Vec); +type StagedFixture<'publisher> = (SealedSegment>, Vec); #[test] fn successful_publication_materializes_only_the_exact_durable_view() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-success")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; assert_eq!(segment_bytes, fixture(SEGMENT_HEX)?); let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let selection = SegmentPublication::one(closed, &segments[0])?; + let selection = publisher.select_segment(sealed, &segments[0])?; let receipt = publish_catalog_generation( &mut publisher, @@ -71,14 +74,15 @@ fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<( let store = StoreFixture::create("catalog-filesystem-retry")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let selection = publisher.select_segment(sealed, &segments[0])?; let first = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::one(closed, &segments[0])?, + selection, &catalog, &segments, )?; @@ -152,18 +156,17 @@ fn restart_policy() -> Result> { )) } -fn stage_one_zero( - publisher: &FilesystemCatalogPublisher, +fn stage_one_zero<'publisher>( + publisher: &'publisher FilesystemCatalogPublisher, store: &StoreFixture, -) -> Result> { +) -> Result, Box> { let stage = publisher.create_segment_stage()?; let record = AdmittedSegmentRecord::for_chunk(&[0])?; - let closed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)? + let sealed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)? .append(record)? - .seal()? - .close(); + .seal()?; let bytes = fs::read(store.staging().join("current.seg"))?; - Ok((closed, bytes)) + Ok((sealed, bytes)) } const fn maximum_segment_policy() -> SegmentReadPolicy { diff --git a/tests/catalog_filesystem_publication/authority_laws.rs b/tests/catalog_filesystem_publication/authority_laws.rs new file mode 100644 index 0000000..c05b47d --- /dev/null +++ b/tests/catalog_filesystem_publication/authority_laws.rs @@ -0,0 +1,102 @@ +use std::error::Error; +use std::fs; +use std::io::{self, Write}; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationPhase, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemWriterLock, SegmentPublication, SegmentRecordLimit, + SegmentStage, StagedSegment, publish_catalog_generation, +}; + +use super::{ + SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, stage_one_zero, +}; + +#[test] +fn metadata_equivalent_external_stage_cannot_authorize_publication() -> Result<(), Box> { + let store = StoreFixture::create("catalog-filesystem-stage-authority")?; + let segment_bytes = fixture(SEGMENT_HEX)?; + fs::write(store.staging().join("current.seg"), &segment_bytes)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let mut staged = StagedSegment::begin(MemoryStage::default(), SegmentRecordLimit::MAXIMUM)?; + for record in segment.records() { + staged = staged.append(record?)?; + } + let segments = [segment]; + let selection = SegmentPublication::one(staged.seal()?.close(), &segments[0])?; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + + let Err(error) = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + selection, + &catalog, + &segments, + ) else { + return Err("external closed-stage metadata authorized publication".into()); + }; + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("external stage reached the wrong refusal boundary".into()); + }; + + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(FilesystemCatalogPublicationError::SegmentAuthorityRequired) + )); + drop(publisher); + store.remove() +} + +#[test] +fn one_publisher_cannot_select_another_publishers_stage() -> Result<(), Box> { + let first_store = StoreFixture::create("catalog-filesystem-stage-owner")?; + let second_store = StoreFixture::create("catalog-filesystem-stage-substitute")?; + let first_lock = FilesystemWriterLock::try_acquire(first_store.path())?; + let first_publisher = FilesystemCatalogPublisher::open(first_lock, restart_policy()?)?; + let second_lock = FilesystemWriterLock::try_acquire(second_store.path())?; + let second_publisher = FilesystemCatalogPublisher::open(second_lock, restart_policy()?)?; + let (sealed, segment_bytes) = stage_one_zero(&first_publisher, &first_store)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + + let Err(error) = second_publisher.select_segment(sealed, &segment) else { + return Err("publisher selected another publisher's sealed stage".into()); + }; + + assert!(matches!( + error, + keep::SegmentPublicationError::PublisherAuthority + )); + drop(first_publisher); + drop(second_publisher); + first_store.remove()?; + second_store.remove() +} + +#[derive(Default)] +struct MemoryStage { + bytes: Vec, +} + +impl Write for MemoryStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index 1d90395..df0573a 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -21,12 +21,12 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box let store = StoreFixture::create("catalog-filesystem-conflict")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; fs::write(store.segment_path(), fixture(EMPTY_SEGMENT_HEX)?)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let selection = SegmentPublication::one(closed, &segments[0])?; + let selection = publisher.select_segment(sealed, &segments[0])?; let error = require_error( publish_catalog_generation( @@ -57,11 +57,11 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< let store = StoreFixture::create("catalog-filesystem-stale")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let selection = SegmentPublication::one(closed, &segments[0])?; + let selection = publisher.select_segment(sealed, &segments[0])?; let _receipt = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), @@ -110,11 +110,11 @@ fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box< fs::write(store.path().join("head.next"), [])?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let selection = SegmentPublication::one(closed, &segments[0])?; + let selection = publisher.select_segment(sealed, &segments[0])?; let error = require_error( publish_catalog_generation( @@ -148,16 +148,17 @@ fn leftover_catalog_stage_refuses_before_segment_pool_mutation() -> Result<(), B let store = StoreFixture::create("catalog-filesystem-catalog-recovery")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; fs::write(store.staging().join("current.cat"), b"recovery evidence")?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let selection = publisher.select_segment(sealed, &segments[0])?; let error = require_error( publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::one(closed, &segments[0])?, + selection, &catalog, &segments, ), @@ -224,14 +225,15 @@ fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box let store = StoreFixture::create("catalog-filesystem-retry-stage")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, segment_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let selection = publisher.select_segment(sealed, &segments[0])?; let _receipt = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::one(closed, &segments[0])?, + selection, &catalog, &segments, )?; @@ -239,14 +241,15 @@ fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; - let (closed, retry_bytes) = stage_one_zero(&publisher, &store)?; + let (sealed, retry_bytes) = stage_one_zero(&publisher, &store)?; let retry_segment = AdmittedSegment::decode(&retry_bytes, maximum_segment_policy())?; let retry_segments = [retry_segment]; + let selection = publisher.select_segment(sealed, &retry_segments[0])?; let error = require_error( publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), - SegmentPublication::one(closed, &retry_segments[0])?, + selection, &catalog, &retry_segments, ), From e3e7c0024d589baf7ccc386bd3020c85c1cfbadd Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 03:42:04 -0700 Subject: [PATCH 29/31] Refuse orphaned pools without a catalog head --- CHANGELOG.md | 6 +- README.md | 9 +- docs/formats/segment-store-v1/publication.md | 6 +- docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/filesystem_catalog_current.rs | 23 ++++ .../filesystem_catalog_publication_error.rs | 10 ++ tests/catalog_filesystem_publication.rs | 2 + .../initialization_laws.rs | 110 ++++++++++++++++++ .../refusal_laws.rs | 60 ++++++---- 9 files changed, 200 insertions(+), 28 deletions(-) create mode 100644 tests/catalog_filesystem_publication/initialization_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c77899f..e1b1c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ after its public API and format compatibility policies are established. that created them. Process-local publisher authority prevents an unrelated metadata-equivalent `ClosedSegment` from authorizing retained `staging/current.seg` bytes. +- First catalog publication now admits an absent `HEAD` only after proving that + both immutable pools are empty. Retained segment or catalog bytes require + explicit recovery and remain untouched. - Filesystem catalog publishers now retain no-follow, read-capable directory handles so required durability synchronization works on Linux instead of failing on `O_PATH` descriptors. @@ -215,7 +218,8 @@ after its public API and format compatibility policies are established. without repeating publication mutations. Retained `head.next` or `current.cat`, an unselected `current.seg`, and every fixed-name stage on an already-current retry now refuse at current-state verification before any - publication mutation. + publication mutation. An absent `HEAD` with any retained segment-pool or + catalog-pool entry also refuses before mutation. - Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact checksummed head, catalog, and segment coordinates; refuses symbolic links, nonregular files, malformed or conflicting bytes, dangling entries, and diff --git a/README.md b/README.md index 10d4428..ac9cdfe 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,11 @@ The durable boundary does not yet initialize or recover a store root. Callers must supply the exact existing `writer.lock`, `staging`, `segments`, and `catalogs` namespace before opening a filesystem publisher. Leftover `head.next`, staged recovery evidence, unknown namespace entries, and -ambiguous crash states remain explicit recovery work in issue #17. Retention, -complete namespace verification, compaction, and garbage collection remain -planned. Presence in the reference CAS does not claim retention, crash -recovery, or durability. +ambiguous crash states remain explicit recovery work in issue #17. An absent +`HEAD` is admitted for first publication only when both immutable pools are +empty. Retention, complete namespace verification, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim +retention, crash recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index df69ce0..b234ca2 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -158,8 +158,10 @@ Issue #16 does not implement store-root initialization or explicit recovery. A caller must supply the exact canonical directories and persistent lock file before opening a publisher. Any retained `head.next` or `current.cat`, and any `current.seg` not owned by the selected staged segment, causes publication to -refuse before mutation and requires issue #17 recovery. An already-current -retry refuses every fixed-name stage. +refuse before mutation and requires issue #17 recovery. When `HEAD` is absent, +the publisher probes both immutable pools and admits first publication only +when both are empty; any entry is preserved as recovery evidence and refuses +the operation. An already-current retry refuses every fixed-name stage. ## Forward publication protocol diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 15d6bd1..8930c3b 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -73,7 +73,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | | `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | -| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | +| `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; an absent head requires empty immutable pools; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | | `KEEP-CATALOG-011` | Every public catalog and publication-head parser boundary is fuzzed from canonical deterministic seeds | Canonical generation-1, generation-2, and bundle artifacts | `fuzz/fuzz_targets/catalog_format.rs`, `xtask/src/fuzz_seed_corpus/catalog_seeds.rs` | Implemented in #16 | diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs index 682abe2..cf56a39 100644 --- a/src/adapters/filesystem_catalog_current.rs +++ b/src/adapters/filesystem_catalog_current.rs @@ -49,6 +49,7 @@ pub(super) fn verify( } } Err(source) if head_is_absent(&source) && expected.current_generation().is_none() => { + require_empty_durable_pools(publisher)?; Ok(CatalogPublicationReadiness::Ready) } Err(source) => Err(filesystem_catalog_artifact::invalid_data(source)), @@ -101,6 +102,28 @@ fn require_no_segment_stage(publisher: &FilesystemCatalogPublisher) -> io::Resul ) } +fn require_empty_durable_pools(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { + require_empty_pool( + &publisher.segments, + FilesystemCatalogPublicationError::SegmentPoolRecoveryRequired, + )?; + require_empty_pool( + &publisher.catalogs, + FilesystemCatalogPublicationError::CatalogPoolRecoveryRequired, + ) +} + +fn require_empty_pool( + directory: &cap_std::fs::Dir, + error: FilesystemCatalogPublicationError, +) -> io::Result<()> { + match directory.entries()?.next() { + None => Ok(()), + Some(Err(source)) => Err(source), + Some(Ok(_entry)) => Err(filesystem_catalog_artifact::invalid_data(error)), + } +} + fn require_absent( directory: &cap_std::fs::Dir, name: &str, diff --git a/src/adapters/filesystem_catalog_publication_error.rs b/src/adapters/filesystem_catalog_publication_error.rs index d8c7507..e4c7a6c 100644 --- a/src/adapters/filesystem_catalog_publication_error.rs +++ b/src/adapters/filesystem_catalog_publication_error.rs @@ -38,6 +38,10 @@ pub enum FilesystemCatalogPublicationError { CatalogRecoveryRequired, /// A leftover `staging/current.seg` requires explicit recovery. SegmentRecoveryRequired, + /// An absent head cannot authorize retained immutable segment bytes. + SegmentPoolRecoveryRequired, + /// An absent head cannot authorize retained immutable catalog bytes. + CatalogPoolRecoveryRequired, } impl fmt::Display for FilesystemCatalogPublicationError { @@ -62,6 +66,12 @@ impl fmt::Display for FilesystemCatalogPublicationError { Self::SegmentRecoveryRequired => { formatter.write_str("current.seg requires explicit recovery") } + Self::SegmentPoolRecoveryRequired => { + formatter.write_str("segment pool requires explicit recovery") + } + Self::CatalogPoolRecoveryRequired => { + formatter.write_str("catalog pool requires explicit recovery") + } } } } diff --git a/tests/catalog_filesystem_publication.rs b/tests/catalog_filesystem_publication.rs index c30a19b..72695d8 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/tests/catalog_filesystem_publication.rs @@ -4,6 +4,8 @@ mod authority_laws; #[path = "catalog_filesystem_publication/directory_laws.rs"] mod directory_laws; +#[path = "catalog_filesystem_publication/initialization_laws.rs"] +mod initialization_laws; #[path = "catalog_filesystem_publication/refusal_laws.rs"] mod refusal_laws; #[path = "segment_filesystem_stage/sandbox.rs"] diff --git a/tests/catalog_filesystem_publication/initialization_laws.rs b/tests/catalog_filesystem_publication/initialization_laws.rs new file mode 100644 index 0000000..3ef866b --- /dev/null +++ b/tests/catalog_filesystem_publication/initialization_laws.rs @@ -0,0 +1,110 @@ +//! This module owns absent-head immutable-pool admission laws. + +use std::error::Error; +use std::fs; +use std::io; + +use keep::{ + CanonicalCatalog, CatalogGeneration, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationPhase, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, + FilesystemWriterLock, SegmentPublication, publish_catalog_generation, +}; + +use super::{CATALOG_HEX, SEGMENT_HEX, StoreFixture, fixture, restart_policy}; + +#[test] +fn absent_head_refuses_a_retained_segment() -> Result<(), Box> { + require_empty_durable_pools(DurablePool::Segments) +} + +#[test] +fn absent_head_refuses_a_retained_catalog() -> Result<(), Box> { + require_empty_durable_pools(DurablePool::Catalogs) +} + +fn require_empty_durable_pools(pool: DurablePool) -> Result<(), Box> { + let store = StoreFixture::create(pool.fixture_name())?; + let artifact = pool.write(&store)?; + let segments = []; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let lock = FilesystemWriterLock::try_acquire(store.path())?; + let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + + let Err(error) = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::none(), + &catalog, + &segments, + ) else { + return Err(format!("absent HEAD admitted retained {} bytes", pool.name()).into()); + }; + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("retained pool artifact reached the wrong refusal boundary".into()); + }; + + assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + let exact = source + .get_ref() + .and_then(|error| error.downcast_ref::()); + assert!(pool.matches(exact)); + drop(publisher); + assert_eq!(fs::read(&artifact)?, pool.bytes()?); + assert!(!store.path().join("HEAD").exists()); + assert!(!store.path().join("head.next").exists()); + assert!(!store.staging().join("current.cat").exists()); + assert!(!store.staging().join("current.seg").exists()); + store.remove() +} + +#[derive(Clone, Copy)] +enum DurablePool { + Segments, + Catalogs, +} + +impl DurablePool { + const fn fixture_name(self) -> &'static str { + match self { + Self::Segments => "catalog-filesystem-orphan-segment", + Self::Catalogs => "catalog-filesystem-orphan-catalog", + } + } + + const fn name(self) -> &'static str { + match self { + Self::Segments => "segment-pool", + Self::Catalogs => "catalog-pool", + } + } + + const fn matches(self, error: Option<&FilesystemCatalogPublicationError>) -> bool { + matches!( + (self, error), + ( + Self::Segments, + Some(FilesystemCatalogPublicationError::SegmentPoolRecoveryRequired) + ) | ( + Self::Catalogs, + Some(FilesystemCatalogPublicationError::CatalogPoolRecoveryRequired) + ) + ) + } + + fn write(self, store: &StoreFixture) -> Result> { + let path = match self { + Self::Segments => store.segment_path(), + Self::Catalogs => store.catalog_path(), + }; + fs::write(path, self.bytes()?)?; + Ok(path.to_path_buf()) + } + + fn bytes(self) -> Result, Box> { + match self { + Self::Segments => fixture(SEGMENT_HEX), + Self::Catalogs => fixture(CATALOG_HEX), + } + } +} diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index df0573a..2668437 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -5,9 +5,9 @@ use std::fs; use keep::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationPhase, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemWriterLock, SegmentPublication, - publish_catalog_generation, + CatalogPublicationExpectation, CatalogPublicationPhase, CatalogRestartError, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemWriterLock, SegmentPublication, publish_catalog_generation, }; use super::{ @@ -21,33 +21,53 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box let store = StoreFixture::create("catalog-filesystem-conflict")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let initial_segments = []; + let initial_catalog = + CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &initial_segments)?; + let _initial_receipt = publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + SegmentPublication::none(), + &initial_catalog, + &initial_segments, + )?; + let initial_head = fs::read(store.path().join("HEAD"))?; + let current = FilesystemCatalogSnapshot::load(store.path(), restart_policy()?)?; + let current_snapshot = current.snapshot()?; + let expectation = CatalogPublicationExpectation::successor_of(¤t_snapshot); let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; - fs::write(store.segment_path(), fixture(EMPTY_SEGMENT_HEX)?)?; + let conflicting_bytes = fixture(EMPTY_SEGMENT_HEX)?; + let conflicting_segment = + AdmittedSegment::decode(&conflicting_bytes, maximum_segment_policy())?; + let conflicting_digest = conflicting_segment.digest(); + fs::write(store.segment_path(), conflicting_bytes)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; + let segment_digest = segment.digest(); let segments = [segment]; - let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let catalog = CanonicalCatalog::from_segments( + CatalogGeneration::new(2)?, + Some(current.catalog_digest()), + &segments, + )?; let selection = publisher.select_segment(sealed, &segments[0])?; let error = require_error( - publish_catalog_generation( - &mut publisher, - CatalogPublicationExpectation::uninitialized(), - selection, - &catalog, - &segments, - ), + publish_catalog_generation(&mut publisher, expectation, selection, &catalog, &segments), "conflicting immutable segment was published", )?; - drop(publisher); - + let CatalogPublicationError::Storage { phase, source } = error else { + return Err("segment conflict reached the wrong refusal boundary".into()); + }; + assert_eq!(phase, CatalogPublicationPhase::VerifySegmentPool); assert!(matches!( - error, - CatalogPublicationError::Storage { - phase: CatalogPublicationPhase::VerifySegmentPool, - .. - } + source + .get_ref() + .and_then(|error| error.downcast_ref::()), + Some(CatalogRestartError::SegmentCoordinate { expected, observed }) + if *expected == segment_digest && *observed == conflicting_digest )); - assert!(!store.path().join("HEAD").exists()); + drop(publisher); + assert_eq!(fs::read(store.path().join("HEAD"))?, initial_head); assert!(store.staging().join("current.seg").exists()); store.remove() } From e9cb1eb7c05653edb5985b19e20dc2a25b5bfef3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 04:06:39 -0700 Subject: [PATCH 30/31] Require platform admission for publishers --- CHANGELOG.md | 7 +++- README.md | 29 +++++++------- docs/formats/segment-store-v1/publication.md | 32 ++++++++++------ docs/formats/segment-store-v1/rationale.md | 21 ++++++++++ docs/formats/segment-store-v1/requirements.md | 8 ++-- src/adapters/filesystem_catalog_publisher.rs | 29 +++++++++++--- .../filesystem_catalog_publisher_tests.rs | 38 +++++++++---------- src/adapters/filesystem_platform_admission.rs | 24 ++++++++++++ .../filesystem_segment_stage_tests.rs | 19 +++++----- src/adapters/mod.rs | 12 ++++++ src/lib.rs | 33 ++++++++-------- .../authority_laws.rs | 13 ++++--- .../directory_laws.rs | 26 +++++++++++-- .../initialization_laws.rs | 5 ++- .../refusal_laws.rs | 28 +++++++++----- tests/segment_filesystem_stage/sandbox.rs | 19 ++++++---- 16 files changed, 237 insertions(+), 106 deletions(-) rename tests/catalog_filesystem_publication.rs => src/adapters/filesystem_catalog_publisher_tests.rs (84%) create mode 100644 src/adapters/filesystem_platform_admission.rs rename tests/segment_filesystem_stage.rs => src/adapters/filesystem_segment_stage_tests.rs (92%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1b1c6b..d79c594 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Filesystem catalog publisher construction now consumes an unforgeable + `FilesystemPlatformAdmission`. No public producer exists until crash-tested + initialization can establish the platform contract in issue #17; acquiring + `FilesystemWriterLock` alone no longer authorizes production construction. - Filesystem segment selection now consumes sealed stages through the publisher that created them. Process-local publisher authority prevents an unrelated metadata-equivalent `ClosedSegment` from authorizing retained @@ -204,7 +208,8 @@ after its public API and format compatibility policies are established. successor proofs; immutable reader snapshots; seeded parser fuzzing; and `BTreeMap` transition-model evidence for `keep.segment-store/v1`. - Blocking `FilesystemCatalogPublisher` publication under a persistent - kernel-managed writer lock, with pinned directory capabilities, + kernel-managed writer lock and required `FilesystemPlatformAdmission`, with + pinned directory capabilities, no-replacement immutable-pool links, complete post-link verification, explicit file and directory synchronization, transitive `head.next` verification, atomic `HEAD` replacement, and stale or recovery-required diff --git a/README.md b/README.md index ac9cdfe..bd11fad 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,12 @@ seal, catalog, and publication-head codecs plus explicit immutable-segment and catalog-generation transitions. `StagedSegment` writes only content-admitted chunk or layout records, while `AdmittedSegment` exposes payloads only after complete framing, checksum, logical-identity, duplicate, and physical-digest -verification. A locked `FilesystemCatalogPublisher` exclusively creates the -fixed `current.seg` stage without truncating existing evidence, and the -`FilesystemSegmentStage` lifetime keeps that writer authority borrowed until -the writable stage closes. +verification. A platform-admitted `FilesystemCatalogPublisher` exclusively +creates the fixed `current.seg` stage without truncating existing evidence, +and the `FilesystemSegmentStage` lifetime keeps that writer authority borrowed +until the writable stage closes. Publisher construction consumes an +unforgeable `FilesystemPlatformAdmission`; no public producer exists until +issue #17 implements crash-tested initialization and platform admission. `FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned root, staging, segment-pool, and catalog-pool capabilities for the complete @@ -54,15 +56,16 @@ logical reads. The reference CAS is executable evidence for M2 storage laws, not a durable backend. Its committed state is process memory; process death loses it all. -The durable boundary does not yet initialize or recover a store root. -Callers must supply the exact existing `writer.lock`, `staging`, `segments`, -and `catalogs` namespace before opening a filesystem publisher. Leftover -`head.next`, staged recovery evidence, unknown namespace entries, and -ambiguous crash states remain explicit recovery work in issue #17. An absent -`HEAD` is admitted for first publication only when both immutable pools are -empty. Retention, complete namespace verification, compaction, and garbage -collection remain planned. Presence in the reference CAS does not claim -retention, crash recovery, or durability. +The durable boundary does not yet initialize, platform-admit, or recover a +store root. Acquiring `FilesystemWriterLock` alone cannot construct a +filesystem publisher. Issue #17 must admit the exact existing `writer.lock`, +`staging`, `segments`, and `catalogs` namespace before it can return +`FilesystemPlatformAdmission`. Leftover `head.next`, staged recovery evidence, +unknown namespace entries, and ambiguous crash states remain explicit recovery +work. An absent `HEAD` is admitted for first publication only when both +immutable pools are empty. Retention, complete namespace verification, +compaction, and garbage collection remain planned. Presence in the reference +CAS does not claim retention, crash recovery, or durability. ```rust use keep::BlobId; diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index b234ca2..3783be7 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -120,10 +120,17 @@ process-scoped exclusion are unsupported. `FilesystemWriterLock::try_acquire` opens the existing regular `writer.lock` without following symbolic links and acquires its exclusive advisory lock -without blocking. `FilesystemCatalogPublisher::open` consumes that authority -and pins the existing store root plus `staging`, `segments`, and `catalogs`. -Both operations perform blocking filesystem I/O. Neither operation initializes, -repairs, enumerates, or removes protocol state. +without blocking. That lock alone cannot construct a publisher. +`FilesystemCatalogPublisher::open` consumes `FilesystemPlatformAdmission`, +which owns the lock after initialization and platform checks, then pins the +existing store root plus `staging`, `segments`, and `catalogs`. Both operations +perform blocking filesystem I/O. Neither operation repairs, enumerates, or +removes protocol state. + +Issue #16 defines the proof type but deliberately exposes no public producer. +The filesystem transition suite uses a crate-private, test-only unchecked proof +to exercise publication mechanics. Issue #17 must implement initialization and +the platform contract before production callers can obtain admission. `publish_catalog_generation` performs complete semantic preflight before the first storage transition. With `FilesystemCatalogPublisher`, it then executes @@ -154,14 +161,15 @@ head-selected coordinates, refuses symbolic links and nonregular artifacts, checks every length before allocation, and reconstructs logical bindings only after all canonical bytes and physical coordinates verify. -Issue #16 does not implement store-root initialization or explicit recovery. A -caller must supply the exact canonical directories and persistent lock file -before opening a publisher. Any retained `head.next` or `current.cat`, and any -`current.seg` not owned by the selected staged segment, causes publication to -refuse before mutation and requires issue #17 recovery. When `HEAD` is absent, -the publisher probes both immutable pools and admits first publication only -when both are empty; any entry is preserved as recovery evidence and refuses -the operation. An already-current retry refuses every fixed-name stage. +Issue #16 does not implement store-root initialization, platform admission, or +explicit recovery. A future admission producer must prove the exact canonical +directories and persistent lock file before opening a publisher. Any retained +`head.next` or `current.cat`, and any `current.seg` not owned by the selected +staged segment, causes publication to refuse before mutation and requires issue +recovery under #17. When `HEAD` is absent, the publisher probes both immutable pools +and admits first publication only when both are empty; any entry is preserved +as recovery evidence and refuses the operation. An already-current retry +refuses every fixed-name stage. ## Forward publication protocol diff --git a/docs/formats/segment-store-v1/rationale.md b/docs/formats/segment-store-v1/rationale.md index eee61f1..9e0d4c0 100644 --- a/docs/formats/segment-store-v1/rationale.md +++ b/docs/formats/segment-store-v1/rationale.md @@ -143,6 +143,27 @@ This deliberately excludes multi-host and filesystems whose advisory locks do not provide the required exclusion. Weakening the lock would violate one-writer publication rather than improve availability. +## Platform admission before publication + +The writer lock proves process-scoped exclusion; it does not prove +case-sensitive names, single-host ownership, hard-link and replacement +semantics, or directory durability. `FilesystemCatalogPublisher::open` +therefore consumes an opaque `FilesystemPlatformAdmission` that owns the +acquired lock. The proof is bound to that exact root authority and cannot be +constructed from metadata or caller assertion. + +Issue #16 exposes no public proof producer. Its crate-internal transition tests +use an explicitly test-only unchecked value, while issue #17 owns the +crash-tested initializer and platform checks that may return production +admission. This staging prevents an incomplete probe from turning successful +syscalls into a durability claim. + +Rejected alternatives were treating `FilesystemWriterLock` as sufficient, +accepting a caller-selected boolean or platform enum, and approving any +filesystem whose individual operations returned success. Each would let an +unsupported platform manufacture the authority that the proof is meant to +represent. + ## Observation before recovery Store opening performs no repair. It produces either one verified reader diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 8930c3b..300dc93 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -59,8 +59,10 @@ retention, or garbage collection. ## Catalog implementation evidence Issue #16 implements catalog-generation admission, writer-locked filesystem -publication, and immutable reader snapshots. Store initialization and explicit -recovery remain owned by issue #17. +publication mechanics, and immutable reader snapshots. Production publisher +construction requires `FilesystemPlatformAdmission`, whose initialization and +platform-checked producer remains owned by issue #17 together with explicit +recovery. @@ -72,7 +74,7 @@ recovery remain owned by issue #17. | `KEEP-CATALOG-004` | Every catalog location equals a verified top-level record span in the exact named segment; construction and admission require every supplied segment to be referenced, and admission scans each referenced segment once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_encoding.rs`, `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | -| `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file | Two-handle lock model | `tests/catalog_writer_lock.rs` | Implemented in #16 | +| `KEEP-CATALOG-007` | One persistent kernel-managed writer lock excludes a second writer without deleting or replacing the lock file; the lock alone cannot construct a publisher without platform admission | Two-handle lock model and construction architecture law | `tests/catalog_writer_lock.rs`, `tests/catalog_filesystem_publication/directory_laws.rs` | Implemented in #16 | | `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; an absent head requires empty immutable pools; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `tests/catalog_filesystem_publication.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs` | Implemented in #16 | diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 9d3976d..8abaa31 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -6,9 +6,9 @@ use cap_std::fs::{Dir, File}; use super::filesystem_publisher_authority::FilesystemPublisherAuthority; use super::{ - AdmittedSegment, CatalogRestartPolicy, ClosedSegment, FilesystemSegmentStage, - FilesystemWriterLock, SealedSegment, SegmentPublication, SegmentPublicationError, - SegmentStageCreateError, sync_capable_directory, + AdmittedSegment, CatalogRestartPolicy, ClosedSegment, FilesystemPlatformAdmission, + FilesystemSegmentStage, FilesystemWriterLock, SealedSegment, SegmentPublication, + SegmentPublicationError, SegmentStageCreateError, sync_capable_directory, }; pub(super) const CURRENT_SEGMENT: &str = "current.seg"; @@ -22,6 +22,10 @@ pub(super) const NEXT_HEAD: &str = "head.next"; /// and catalog-pool directory capabilities until it is dropped. Dropping it /// closes open stages and directory capabilities before releasing the writer /// lock, but never publishes, removes, truncates, or repairs protocol state. +/// +/// Construction consumes a [`FilesystemPlatformAdmission`] proof. Keep exposes +/// no public producer for that proof until issue #17 supplies crash-tested +/// initialization and platform admission. #[must_use] pub struct FilesystemCatalogPublisher { pub(super) root: Dir, @@ -38,7 +42,7 @@ pub struct FilesystemCatalogPublisher { } impl FilesystemCatalogPublisher { - /// Pins the canonical publication directories under an acquired writer lock. + /// Pins canonical publication directories under admitted writer authority. /// /// # Errors /// @@ -47,7 +51,11 @@ impl FilesystemCatalogPublisher { /// returns [`io::ErrorKind::NotADirectory`]. A failure drops `lock` and /// therefore releases writer authority. Success allocates one ephemeral /// authority token that binds later stage selection to this publisher. - pub fn open(lock: FilesystemWriterLock, policy: CatalogRestartPolicy) -> io::Result { + pub fn open( + admission: FilesystemPlatformAdmission, + policy: CatalogRestartPolicy, + ) -> io::Result { + let lock = admission.into_lock(); let pinned_root = lock.clone_directory()?; let root = sync_capable_directory::open(&pinned_root, ".")?; let staging = sync_capable_directory::open(&root, "staging")?; @@ -66,6 +74,17 @@ impl FilesystemCatalogPublisher { }) } + #[cfg(test)] + pub(super) fn open_unchecked_for_tests( + lock: FilesystemWriterLock, + policy: CatalogRestartPolicy, + ) -> io::Result { + Self::open( + FilesystemPlatformAdmission::unchecked_for_tests(lock), + policy, + ) + } + /// Exclusively creates `staging/current.seg` under this writer authority. /// /// The returned stage borrows this publisher until the stage is dropped or diff --git a/tests/catalog_filesystem_publication.rs b/src/adapters/filesystem_catalog_publisher_tests.rs similarity index 84% rename from tests/catalog_filesystem_publication.rs rename to src/adapters/filesystem_catalog_publisher_tests.rs index 72695d8..4319f39 100644 --- a/tests/catalog_filesystem_publication.rs +++ b/src/adapters/filesystem_catalog_publisher_tests.rs @@ -1,22 +1,21 @@ //! Filesystem-backed catalog publication laws. -#[path = "catalog_filesystem_publication/authority_laws.rs"] +#[path = "../../tests/catalog_filesystem_publication/authority_laws.rs"] mod authority_laws; -#[path = "catalog_filesystem_publication/directory_laws.rs"] +#[path = "../../tests/catalog_filesystem_publication/directory_laws.rs"] mod directory_laws; -#[path = "catalog_filesystem_publication/initialization_laws.rs"] +#[path = "../../tests/catalog_filesystem_publication/initialization_laws.rs"] mod initialization_laws; -#[path = "catalog_filesystem_publication/refusal_laws.rs"] +#[path = "../../tests/catalog_filesystem_publication/refusal_laws.rs"] mod refusal_laws; -#[path = "segment_filesystem_stage/sandbox.rs"] -pub mod sandbox; -mod support; use std::error::Error; use std::fs; use std::path::{Path, PathBuf}; -use keep::{ +use super::filesystem_test_sandbox::TestDirectory; +use super::test_support::decode_hex; +use crate::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, @@ -24,24 +23,23 @@ use keep::{ SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, publish_catalog_generation, }; -use sandbox::TestDirectory; -use support::decode_hex; -const SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); -const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/empty-segment.hex"); -const CATALOG_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); -const HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const SEGMENT_HEX: &str = include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/empty-segment.hex"); +const CATALOG_HEX: &str = include_str!("../../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../../conformance/segment-store/v1/one-zero-head.hex"); const CATALOG_DIGEST: &str = "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320"; const SEGMENT_DIGEST: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc"; const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; - type StagedFixture<'publisher> = (SealedSegment>, Vec); #[test] fn successful_publication_materializes_only_the_exact_durable_view() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-success")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; assert_eq!(segment_bytes, fixture(SEGMENT_HEX)?); let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; @@ -75,7 +73,8 @@ fn successful_publication_materializes_only_the_exact_durable_view() -> Result<( fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-retry")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; @@ -91,7 +90,8 @@ fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<( drop(publisher); let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let retry = publish_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), @@ -182,7 +182,7 @@ fn fixture(hex: &str) -> Result, Box> { #[test] fn publisher_drop_closes_writable_stages_before_releasing_writer_authority() -> Result<(), Box> { - let source = include_str!("../src/adapters/filesystem_catalog_publisher.rs"); + let source = include_str!("filesystem_catalog_publisher.rs"); let catalog_stage = source .find("pub(super) catalog_stage:") .ok_or("publisher must retain the catalog stage")?; diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs new file mode 100644 index 0000000..013533e --- /dev/null +++ b/src/adapters/filesystem_platform_admission.rs @@ -0,0 +1,24 @@ +//! This module owns proof that a filesystem root passed platform admission. + +use super::FilesystemWriterLock; + +/// Exclusive writer authority over a platform-admitted filesystem root. +/// +/// Fields are private so only Keep's initialization and platform-admission +/// boundary can create production values. That boundary remains intentionally +/// absent until issue #17 supplies its crash-tested implementation. +#[must_use] +pub struct FilesystemPlatformAdmission { + lock: FilesystemWriterLock, +} + +impl FilesystemPlatformAdmission { + #[cfg(test)] + pub(super) const fn unchecked_for_tests(lock: FilesystemWriterLock) -> Self { + Self { lock } + } + + pub(super) fn into_lock(self) -> FilesystemWriterLock { + self.lock + } +} diff --git a/tests/segment_filesystem_stage.rs b/src/adapters/filesystem_segment_stage_tests.rs similarity index 92% rename from tests/segment_filesystem_stage.rs rename to src/adapters/filesystem_segment_stage_tests.rs index 470bd8f..08369fc 100644 --- a/tests/segment_filesystem_stage.rs +++ b/src/adapters/filesystem_segment_stage_tests.rs @@ -1,24 +1,21 @@ //! Exclusive filesystem segment-stage creation laws. -#[path = "segment_filesystem_stage/sandbox.rs"] -pub mod sandbox; -mod support; - use std::error::Error; use std::fs; use std::io::ErrorKind; -use keep::{ +use super::filesystem_test_sandbox::TestDirectory; +use super::test_support::decode_hex; +use crate::{ AdmittedSegmentRecord, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, FilesystemWriterLock, LayoutEntryLimit, SegmentHeader, SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, StagedSegment, }; -use sandbox::TestDirectory; -use support::decode_hex; const ONE_ZERO_SEGMENT_HEX: &str = - include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); -const EMPTY_SEGMENT_HEX: &str = include_str!("../conformance/segment-store/v1/empty-segment.hex"); + include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/empty-segment.hex"); #[test] fn exclusive_creation_never_truncates_existing_stage() -> Result<(), Box> { @@ -121,5 +118,7 @@ fn open_publisher(sandbox: &TestDirectory) -> Result Result<( let selection = SegmentPublication::one(staged.seal()?.close(), &segments[0])?; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let Err(error) = publish_catalog_generation( &mut publisher, @@ -59,9 +60,11 @@ fn one_publisher_cannot_select_another_publishers_stage() -> Result<(), Box Result<(), Box Result<(), Box> { + let publisher = include_str!("../../src/adapters/filesystem_catalog_publisher.rs"); + let admission = include_str!("../../src/adapters/filesystem_platform_admission.rs"); + if !publisher.contains("pub fn open(\n admission: FilesystemPlatformAdmission,") { + return Err("publisher construction does not require platform admission".into()); + } + let public_items = admission + .lines() + .map(str::trim_start) + .filter(|line| line.starts_with("pub ")) + .collect::>(); + if public_items != ["pub struct FilesystemPlatformAdmission {"] { + return Err("platform admission exposes an unverified public producer".into()); + } + Ok(()) +} + #[test] fn publisher_refuses_a_non_directory_protocol_namespace() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-nondirectory")?; @@ -13,7 +31,8 @@ fn publisher_refuses_a_non_directory_protocol_namespace() -> Result<(), Box Result<(), Box Result<(), Box> let segments = []; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let Err(error) = publish_catalog_generation( &mut publisher, diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index 2668437..4aaf5f8 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -3,24 +3,25 @@ use std::error::Error; use std::fs; -use keep::{ +use crate::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationPhase, CatalogRestartError, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemWriterLock, SegmentPublication, publish_catalog_generation, }; +use super::super::test_support::require_error; use super::{ EMPTY_SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, stage_one_zero, }; -use crate::support::require_error; #[test] fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-conflict")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let initial_segments = []; let initial_catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &initial_segments)?; @@ -76,7 +77,8 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-stale")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; @@ -93,7 +95,8 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< let stale_candidate = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &[])?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let error = require_error( publish_catalog_generation( &mut publisher, @@ -129,7 +132,8 @@ fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box< let store = StoreFixture::create("catalog-filesystem-next-head")?; fs::write(store.path().join("head.next"), [])?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; @@ -167,7 +171,8 @@ fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box< fn leftover_catalog_stage_refuses_before_segment_pool_mutation() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-catalog-recovery")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; fs::write(store.staging().join("current.cat"), b"recovery evidence")?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; @@ -212,7 +217,8 @@ fn catalog_only_publication_refuses_a_leftover_segment_stage() -> Result<(), Box fs::write(&stage, b"recovery evidence")?; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &[])?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let error = require_error( publish_catalog_generation( &mut publisher, @@ -244,7 +250,8 @@ fn catalog_only_publication_refuses_a_leftover_segment_stage() -> Result<(), Box fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-retry-stage")?; let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; @@ -260,7 +267,8 @@ fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box drop(publisher); let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(lock, restart_policy()?)?; + let mut publisher = + FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; let (sealed, retry_bytes) = stage_one_zero(&publisher, &store)?; let retry_segment = AdmittedSegment::decode(&retry_bytes, maximum_segment_policy())?; let retry_segments = [retry_segment]; diff --git a/tests/segment_filesystem_stage/sandbox.rs b/tests/segment_filesystem_stage/sandbox.rs index 094db57..a95cd3e 100644 --- a/tests/segment_filesystem_stage/sandbox.rs +++ b/tests/segment_filesystem_stage/sandbox.rs @@ -1,11 +1,11 @@ -//! Deterministic filesystem sandbox for segment-stage integration laws. +//! Deterministic filesystem sandbox for filesystem adapter laws. use std::fs; use std::io; use std::path::{Path, PathBuf}; -/// Process-isolated directory beneath Cargo's integration-test scratch root. -pub struct TestDirectory { +/// Process-isolated directory beneath Cargo's test scratch root. +pub(super) struct TestDirectory { path: PathBuf, } @@ -16,9 +16,12 @@ impl TestDirectory { /// /// Returns the exact filesystem failure from scratch-root creation, /// removal of same-process stale test evidence, or sandbox creation. - pub fn create(name: &str) -> io::Result { - let root = Path::new(env!("CARGO_TARGET_TMPDIR")); - fs::create_dir_all(root)?; + pub(super) fn create(name: &str) -> io::Result { + let root = option_env!("CARGO_TARGET_TMPDIR").map_or_else( + || Path::new(env!("CARGO_MANIFEST_DIR")).join("target/tmp"), + PathBuf::from, + ); + fs::create_dir_all(&root)?; let path = root.join(format!("keep-{name}-{}", std::process::id())); match fs::remove_dir_all(&path) { Ok(()) => {} @@ -30,7 +33,7 @@ impl TestDirectory { } /// Returns the sandbox root. - pub fn path(&self) -> &Path { + pub(super) fn path(&self) -> &Path { &self.path } @@ -39,7 +42,7 @@ impl TestDirectory { /// # Errors /// /// Returns the exact recursive-removal filesystem failure. - pub fn remove(self) -> io::Result<()> { + pub(super) fn remove(self) -> io::Result<()> { fs::remove_dir_all(self.path) } } From 4bd62c8a792168bbfdc22f1f37979b37c90b91b2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 04:12:08 -0700 Subject: [PATCH 31/31] Verify catalog integrity before entry semantics --- CHANGELOG.md | 3 +++ docs/formats/segment-store-v1/catalog.md | 5 +++++ docs/formats/segment-store-v1/requirements.md | 2 +- src/adapters/catalog_decoder.rs | 2 +- tests/catalog/integrity_laws.rs | 20 ++++++++++++++++++- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d79c594..2b0e9d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ after its public API and format compatibility policies are established. ### Changed +- Catalog decoding now verifies the catalog checksum and physical digest before + interpreting entry semantics. Corrupt identity-bearing bytes therefore fail + at the integrity boundary instead of producing a semantic entry error. - Filesystem catalog publisher construction now consumes an unforgeable `FilesystemPlatformAdmission`. No public producer exists until crash-tested initialization can establish the platform contract in issue #17; acquiring diff --git a/docs/formats/segment-store-v1/catalog.md b/docs/formats/segment-store-v1/catalog.md index 3a612bd..9f9d26c 100644 --- a/docs/formats/segment-store-v1/catalog.md +++ b/docs/formats/segment-store-v1/catalog.md @@ -103,6 +103,11 @@ catalog_digest = framed_blake3_v1( ) ``` +After admitting the fixed header and exact declared length, decoding verifies +both trailer fields before interpreting any entry semantics. Covered-byte +corruption therefore returns a checksum or digest disagreement even when the +same bytes would also violate an entry field invariant. + The catalog digest is a physical generation coordinate and predecessor witness. It does not establish retention or application history. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 300dc93..60d4ae8 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -69,7 +69,7 @@ recovery. | ID | Implemented requirement | Oracle | Executable evidence | Status | | --- | --- | --- | --- | --- | | `KEEP-CATALOG-001` | `CatalogGeneration` admits positive values and refuses overflow when deriving a successor | Checked scalar model | `tests/catalog_generation.rs` | Implemented in #16 | -| `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes | Independent golden corpus | `tests/catalog.rs`, `tests/publication_head.rs` | Implemented in #16 | +| `KEEP-CATALOG-002` | Catalog and publication-head codecs reproduce every frozen version-1 artifact and refuse noncanonical bytes; catalog checksum and digest admission precede entry semantics | Independent golden corpus and mutation precedence oracle | `tests/catalog.rs`, `tests/catalog/integrity_laws.rs`, `tests/publication_head.rs` | Implemented in #16 | | `KEEP-CATALOG-003` | Catalog entries are sorted by logical identity and duplicate keys are refused independently of input order | Ordered reference map | `tests/catalog_ordering.rs` | Implemented in #16 | | `KEEP-CATALOG-004` | Every catalog location equals a verified top-level record span in the exact named segment; construction and admission require every supplied segment to be referenced, and admission scans each referenced segment once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_encoding.rs`, `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | diff --git a/src/adapters/catalog_decoder.rs b/src/adapters/catalog_decoder.rs index 8091141..b1127f1 100644 --- a/src/adapters/catalog_decoder.rs +++ b/src/adapters/catalog_decoder.rs @@ -16,8 +16,8 @@ pub(super) fn decode(encoded: &[u8]) -> Result, CatalogDe let fields = catalog_header_decoder::decode(encoded)?; let metadata = validate_header(&fields)?; validate_observed_length(encoded, metadata.length())?; - catalog_entry_sequence::validate(encoded, metadata.entry_count())?; let digest = catalog_integrity::validate(encoded)?; + catalog_entry_sequence::validate(encoded, metadata.entry_count())?; Ok(ChecksummedCatalog::from_verified_parts( encoded, metadata, diff --git a/tests/catalog/integrity_laws.rs b/tests/catalog/integrity_laws.rs index 3e776da..fd926cd 100644 --- a/tests/catalog/integrity_laws.rs +++ b/tests/catalog/integrity_laws.rs @@ -4,9 +4,27 @@ use std::error::Error; use keep::{CatalogDecodeError, ChecksummedCatalog}; -use super::{GENERATION_ONE_HEX, catalog_bytes}; +use super::{FIRST_ENTRY_OFFSET, GENERATION_ONE_HEX, catalog_bytes}; use crate::support::require_error; +#[test] +fn catalog_integrity_precedes_entry_semantics() -> Result<(), Box> { + let mut encoded = catalog_bytes(GENERATION_ONE_HEX)?; + *encoded + .get_mut(FIRST_ENTRY_OFFSET) + .ok_or("catalog lacks its first entry")? = 3; + + let error = require_error( + ChecksummedCatalog::decode(&encoded), + "catalog with stale integrity fields was admitted", + )?; + assert!( + matches!(error, CatalogDecodeError::ChecksumMismatch { .. }), + "unexpected refusal: {error:?}" + ); + Ok(()) +} + #[test] fn catalog_refuses_wrong_width_checksum_and_digest() -> Result<(), Box> { let mut truncated = catalog_bytes(GENERATION_ONE_HEX)?;