diff --git a/README.md b/README.md index a218f84..cd97779 100644 --- a/README.md +++ b/README.md @@ -341,7 +341,10 @@ ByteStorage envelope vectors from `*_bin` twins (msgpack `bin`, canonical for 1.1+ writers): every vector in both sets must decode to the exact payload bytes, while re-encode byte-identity is asserted against the set matching this crate's current -writer encoding (legacy, until the `serde_bytes` writer flip lands). +writer encoding — msgpack `bin`, since the protocol 1.1 `serde_bytes` +writer flip (LAB-866; `checksum` deliberately stays array-of-ints per the +protocol's normative scope exclusion). Legacy envelopes remain readable +forever. `tests/dual_decode.rs` proves both reader shapes accept both encodings, including bin16/bin32 width headers. The fixture is vendored at `tests/vectors/wire-format.json` and integrity-pinned by sha256 — to update diff --git a/benches/hot_path.rs b/benches/hot_path.rs index 5c746d9..6b99ea5 100644 --- a/benches/hot_path.rs +++ b/benches/hot_path.rs @@ -7,7 +7,7 @@ //! paths are identified. Sizes chosen to span the realistic cache-payload //! distribution (64B keys, 1KB values, 64KB large objects). -use cachekit_core::{ByteStorage, ZeroKnowledgeEncryptor}; +use cachekit_core::{ByteStorage, StorageEnvelope, ZeroKnowledgeEncryptor}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; const SIZES: &[usize] = &[64, 256, 1024, 4 * 1024, 16 * 1024, 64 * 1024]; @@ -16,6 +16,64 @@ fn make_payload(size: usize) -> Vec { (0..size).map(|i| (i % 256) as u8).collect() } +/// Deterministic xorshift64* stream — incompressible payload without a rand +/// dependency (same generator as tests/dual_decode.rs). +fn incompressible(len: usize) -> Vec { + let mut state: u64 = 0x9e3779b97f4a7c15; + let mut out = Vec::with_capacity(len + 8); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.wrapping_mul(0x2545f4914f6cdd1d).to_le_bytes()); + } + out.truncate(len); + out +} + +/// Protocol 1.1 bin-encoding proof workload (LAB-764 / LAB-866): 64 MiB +/// incompressible payload, where compressed_data dominates the envelope and +/// the array-of-ints vs msgpack-bin difference is fully visible. Measures the +/// envelope codec in isolation (rmp encode/decode of StorageEnvelope) and the +/// full store()/retrieve() e2e paths. +fn bench_envelope_codec_64mib(c: &mut Criterion) { + const SIZE: usize = 64 * 1024 * 1024; + let storage = ByteStorage::new(None); + let payload = incompressible(SIZE); + let wire = storage.store(&payload, None).unwrap(); + let envelope: StorageEnvelope = rmp_serde::from_slice(&wire).unwrap(); + // Self-enforcing workload check: LZ4 overhead is positive on incompressible + // input, so a generator regression toward compressible data fails here + // instead of silently benchmarking the wrong workload. + assert!( + envelope.compressed_data.len() >= SIZE, + "bench payload must be incompressible" + ); + eprintln!( + "64mib_incompressible: compressed_data {} B, envelope wire {} B (ratio {:.4})", + envelope.compressed_data.len(), + wire.len(), + wire.len() as f64 / envelope.compressed_data.len() as f64 + ); + + let mut group = c.benchmark_group("byte_storage/64mib_incompressible"); + group.sample_size(10); + group.throughput(Throughput::Bytes(SIZE as u64)); + group.bench_function("envelope_encode", |b| { + b.iter(|| black_box(rmp_serde::to_vec(black_box(&envelope)).unwrap())); + }); + group.bench_function("envelope_decode", |b| { + b.iter(|| black_box(rmp_serde::from_slice::(black_box(&wire)).unwrap())); + }); + group.bench_function("store_e2e", |b| { + b.iter(|| black_box(storage.store(black_box(&payload), None).unwrap())); + }); + group.bench_function("retrieve_e2e", |b| { + b.iter(|| black_box(storage.retrieve(black_box(&wire)).unwrap())); + }); + group.finish(); +} + fn bench_byte_storage_roundtrip(c: &mut Criterion) { let storage = ByteStorage::new(None); let mut group = c.benchmark_group("byte_storage/roundtrip"); @@ -54,5 +112,10 @@ fn bench_encrypt_decrypt(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_byte_storage_roundtrip, bench_encrypt_decrypt); +criterion_group!( + benches, + bench_byte_storage_roundtrip, + bench_encrypt_decrypt, + bench_envelope_codec_64mib +); criterion_main!(benches); diff --git a/src/byte_storage.rs b/src/byte_storage.rs index 8b377a6..87351cf 100644 --- a/src/byte_storage.rs +++ b/src/byte_storage.rs @@ -54,6 +54,11 @@ const MAX_COMPRESSION_RATIO: u64 = 1000; #[derive(Serialize, Deserialize)] pub struct StorageEnvelope { /// Compressed payload data + /// + /// Serialized as msgpack `bin` (protocol 1.1, `envelope-bin-encoding.md`). + /// Scope is strictly this field: `checksum` stays array-of-ints by + /// normative exclusion (crypto-adjacent surface — MUST NOT flip). + #[serde(with = "serde_bytes")] pub compressed_data: Vec, /// xxHash3-64 checksum for integrity (8 bytes) pub checksum: [u8; 8], diff --git a/tests/dual_decode.rs b/tests/dual_decode.rs index 862dcc4..598aa8e 100644 --- a/tests/dual_decode.rs +++ b/tests/dual_decode.rs @@ -2,45 +2,42 @@ //! //! Protocol 1.1 (`protocol/decisions/envelope-bin-encoding.md`) flips //! `compressed_data` from the legacy msgpack array-of-integers encoding to -//! msgpack `bin` — readers first. This file is the readers-first CI proof: -//! under the locked decoder stack (rmp-serde + serde_bytes), BOTH reader -//! shapes accept BOTH encodings, so `bin` envelopes written by 1.1+ writers -//! are readable today and legacy envelopes stay readable forever. +//! msgpack `bin` — readers first, then the writer (LAB-866, shipped). This +//! file is the permanent CI proof: under the locked decoder stack +//! (rmp-serde with serde_bytes), BOTH reader shapes accept BOTH encodings, +//! so `bin` envelopes from 1.1+ writers are readable by pre-1.1 readers and +//! legacy envelopes stay readable forever. //! //! The full matrix, executed against the byte-pinned protocol vectors: //! -//! | Reader | legacy wire (array) | new wire (bin) | -//! |------------------------------------|---------------------|----------------| -//! | old (plain `Vec`, shipped) | status quo | proven here | -//! | …incl. `ByteStorage::retrieve()` | | proven here | -//! | new (`serde_bytes`, writer-flip) | proven here | proven here | +//! | Reader | legacy wire (array) | new wire (bin) | +//! |-------------------------------------|---------------------|----------------| +//! | legacy (plain `Vec`, pre-1.1) | proven here | proven here | +//! | shipped (`serde_bytes`, 1.1 writer) | proven here | proven here | +//! | …incl. `ByteStorage::retrieve()` | proven here | proven here | //! //! Seeded from the LAB-764 toolchain experiment (`dual_decode_experiment.rs`, -//! attached to the decision memo on that issue); the throughput half of the -//! experiment is deliberately absent — benchmark evidence belongs to the -//! writer-flip stage (LAB-510 harness), not this test-only release. +//! attached to the decision memo on that issue); throughput evidence lives in +//! the LAB-510 harness (`benches/hot_path.rs`, 64 MiB incompressible group). //! //! Width boundaries: the fixture's `*_bin` twins all fit in a bin8 header //! (`0xc4`, ≤ 255 B). The `width_boundary_*` tests cover the bin16 (`0xc5`) -//! and bin32 (`0xc6`) headers with real LZ4 output, per the MUST recorded in -//! the decision record — fixture-level width vectors land with the writer -//! flip, where the real writer can generate them natively. +//! and bin32 (`0xc6`) headers with real LZ4 output from the real writer, per +//! the MUST recorded in the decision record. #![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] use cachekit_core::{ByteStorage, StorageEnvelope}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; const FIXTURE: &str = include_str!("vectors/wire-format.json"); -/// Post-writer-flip twin of StorageEnvelope: identical shape, `serde_bytes` -/// on `compressed_data` only. `checksum: [u8; 8]` deliberately stays -/// array-of-ints — normative scope exclusion in the decision record -/// (crypto-adjacent surface; implementations MUST NOT flip it -/// opportunistically). -#[derive(Serialize, Deserialize, PartialEq, Debug)] -struct StorageEnvelopeV2 { - #[serde(with = "serde_bytes")] +/// Pre-1.1 reader stand-in: identical shape to StorageEnvelope but with a +/// plain `Vec` (no `serde_bytes`) on `compressed_data` — the shape every +/// pre-writer-flip SDK shipped. Kept forever: these matrix legs are the proof +/// that legacy readers accept `bin` wire and that legacy wire still decodes. +#[derive(Deserialize, PartialEq, Debug)] +struct StorageEnvelopeLegacy { compressed_data: Vec, checksum: [u8; 8], original_size: u32, @@ -67,17 +64,17 @@ fn load_vectors() -> Vec { } /// Assert one envelope's bytes decode identically through every reader shape: -/// old struct, new struct, and the full shipped `retrieve()` path (checksum -/// validation + decompression-ratio guard included). +/// legacy struct, shipped struct, and the full shipped `retrieve()` path +/// (checksum validation + decompression-ratio guard included). fn assert_all_readers_decode(name: &str, wire: &[u8], expected_payload: &[u8]) { - let old: StorageEnvelope = rmp_serde::from_slice(wire) - .unwrap_or_else(|e| panic!("[{name}] old reader (plain Vec) failed: {e}")); - let new: StorageEnvelopeV2 = rmp_serde::from_slice(wire) - .unwrap_or_else(|e| panic!("[{name}] new reader (serde_bytes) failed: {e}")); - assert_eq!(old.compressed_data, new.compressed_data, "[{name}]"); - assert_eq!(old.checksum, new.checksum, "[{name}]"); - assert_eq!(old.original_size, new.original_size, "[{name}]"); - assert_eq!(old.format, new.format, "[{name}]"); + let legacy: StorageEnvelopeLegacy = rmp_serde::from_slice(wire) + .unwrap_or_else(|e| panic!("[{name}] legacy reader (plain Vec) failed: {e}")); + let shipped: StorageEnvelope = rmp_serde::from_slice(wire) + .unwrap_or_else(|e| panic!("[{name}] shipped reader (serde_bytes) failed: {e}")); + assert_eq!(legacy.compressed_data, shipped.compressed_data, "[{name}]"); + assert_eq!(legacy.checksum, shipped.checksum, "[{name}]"); + assert_eq!(legacy.original_size, shipped.original_size, "[{name}]"); + assert_eq!(legacy.format, shipped.format, "[{name}]"); let storage = ByteStorage::new(None); let (payload, _format) = storage @@ -90,9 +87,9 @@ fn assert_all_readers_decode(name: &str, wire: &[u8], expected_payload: &[u8]) { } /// The 4-cell matrix over the legacy pinned vectors: decode each with both -/// reader shapes, re-encode through the new (serde_bytes) shape to produce -/// bin wire, and prove that bin wire decodes through both reader shapes AND -/// today's shipped `retrieve()`. +/// reader shapes, re-encode through the shipped (serde_bytes) writer shape to +/// produce bin wire, and prove that bin wire decodes through both reader +/// shapes AND the shipped `retrieve()`. #[test] fn dual_decode_matrix_against_legacy_vectors() { let vectors = load_vectors(); @@ -112,11 +109,11 @@ fn dual_decode_matrix_against_legacy_vectors() { // (the readers-first claim), plus retrieve() on the pinned bytes. assert_all_readers_decode(&v.name, &old_wire, &input); - // Produce new wire the way the flipped writer will. - let new_from_old: StorageEnvelopeV2 = rmp_serde::from_slice(&old_wire) - .unwrap_or_else(|e| panic!("[{}] new reader must decode legacy wire: {e}", v.name)); + // Produce new wire the way the shipped writer does. + let new_from_old: StorageEnvelope = rmp_serde::from_slice(&old_wire) + .unwrap_or_else(|e| panic!("[{}] shipped reader must decode legacy wire: {e}", v.name)); let new_wire = rmp_serde::to_vec(&new_from_old) - .unwrap_or_else(|e| panic!("[{}] serde_bytes shape must serialize: {e}", v.name)); + .unwrap_or_else(|e| panic!("[{}] shipped shape must serialize: {e}", v.name)); // compressed_data (element [0], right after the 0x94 fixarray marker) // must now carry a bin marker. @@ -141,8 +138,9 @@ fn dual_decode_matrix_against_legacy_vectors() { // 3 B array16 header, so bin8's 2 B header shrinks it. Measured worst // case on the pinned set is exactly +1 (`empty`, 25 -> 26 B); every // other vector ties or shrinks. This +1 B ceiling is the micro- - // regression contract the writer-flip step (LAB-764) will read, so keep - // it tight — a +2/+3 bloat regression must not slip through green. + // regression contract accepted by the writer flip (LAB-764/LAB-866), + // so keep it tight — a +2/+3 bloat regression must not slip through + // green. assert!( new_wire.len() <= old_wire.len() + 1, "[{}] new wire exceeds the +1 B bin8-header ceiling", @@ -155,11 +153,10 @@ fn dual_decode_matrix_against_legacy_vectors() { } } -/// The pinned `*_bin` twins through the same matrix, plus the writer-flip -/// forecast: re-encoding a twin through the serde_bytes shape must be -/// byte-identical to the protocol's pinned bin bytes — the writer-flip diff -/// will point `vectors_reencode_byte_identical` at this set, and this assert -/// guarantees that switch lands green. +/// The pinned `*_bin` twins through the same reader matrix. Re-encode +/// byte-identity for this set lives in `tests/wire_format_vectors.rs` +/// (`vectors_reencode_byte_identical` at store() level, +/// `envelope_codec_roundtrip_byte_identical` at codec level). #[test] fn dual_decode_matrix_against_bin_vectors() { let vectors = load_vectors(); @@ -176,17 +173,6 @@ fn dual_decode_matrix_against_bin_vectors() { .unwrap_or_else(|e| panic!("[{}] input_hex must decode: {e}", v.name)); assert_all_readers_decode(&v.name, &wire, &input); - - let env: StorageEnvelopeV2 = rmp_serde::from_slice(&wire) - .unwrap_or_else(|e| panic!("[{}] new reader must decode bin wire: {e}", v.name)); - let reencoded = rmp_serde::to_vec(&env) - .unwrap_or_else(|e| panic!("[{}] serde_bytes shape must serialize: {e}", v.name)); - assert_eq!( - hex::encode(&reencoded), - hex::encode(&wire), - "[{}] serde_bytes re-encode is not byte-identical to the pinned bin vector", - v.name - ); } } @@ -206,22 +192,16 @@ fn incompressible(len: usize) -> Vec { } /// Build a real envelope (real LZ4 output) from an incompressible payload, -/// bin-encode it via the serde_bytes shape, and assert the expected bin width -/// marker before running the full reader matrix on it. +/// bin-encode it via the shipped writer shape, and assert the expected bin +/// width marker before running the full reader matrix on it. fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) { let payload = incompressible(payload_len); let env = StorageEnvelope::new(&payload, "msgpack".to_string()) .unwrap_or_else(|e| panic!("[{name}] envelope construction failed: {e:?}")); let compressed_len = env.compressed_data.len(); - let v2 = StorageEnvelopeV2 { - compressed_data: env.compressed_data, - checksum: env.checksum, - original_size: env.original_size, - format: env.format, - }; - let wire = rmp_serde::to_vec(&v2) - .unwrap_or_else(|e| panic!("[{name}] serde_bytes shape must serialize: {e}")); + let wire = rmp_serde::to_vec(&env) + .unwrap_or_else(|e| panic!("[{name}] shipped shape must serialize: {e}")); assert_eq!(wire[0], 0x94, "[{name}] outer fixarray(4) preserved"); assert_eq!( wire[1], expected_marker, diff --git a/tests/wire_format_vectors.rs b/tests/wire_format_vectors.rs index 1eae4be..03698cf 100644 --- a/tests/wire_format_vectors.rs +++ b/tests/wire_format_vectors.rs @@ -12,8 +12,8 @@ //! their `*_bin` twins (`"envelope_encoding": "bin"`, canonical for 1.1+ //! writers). Decode-identity runs against BOTH sets, forever. Re-encode //! byte-identity runs against the set matching this crate's CURRENT writer -//! encoding — legacy today; the writer-flip diff (`serde_bytes` on -//! `compressed_data`) switches those assertions to the `*_bin` set. +//! encoding — `bin` since the writer flip (`serde_bytes` on +//! `compressed_data`, LAB-866), so those assertions target the `*_bin` set. //! //! Fixture provenance: vendored from //! `test-vectors/wire-format.json` @@ -63,8 +63,8 @@ struct Vector { } impl Vector { - /// Encoded with the legacy array-of-integers `compressed_data` — the - /// encoding this crate's writer currently emits. + /// Encoded with the legacy array-of-integers `compressed_data` — what + /// pre-1.1 writers emitted; retained forever as legacy-read proof. fn is_legacy(&self) -> bool { self.envelope_encoding.is_none() } @@ -165,13 +165,17 @@ fn vectors_decode_to_expected_payload() { /// and cross-version reads are at risk — regenerate vectors deliberately in /// the protocol repo, never adjust expectations here. /// -/// Legacy vectors only: this asserts against the encoding the writer CURRENTLY -/// emits. The writer-flip diff switches this filter to the `*_bin` set in the -/// same PR that adds `serde_bytes` to `compressed_data`. +/// `*_bin` vectors only: this asserts against the encoding the writer +/// CURRENTLY emits — msgpack `bin` since the writer flip (`serde_bytes` on +/// `compressed_data`, LAB-866). #[test] fn vectors_reencode_byte_identical() { let storage = ByteStorage::new(None); - for vector in load_fixture().vectors.into_iter().filter(Vector::is_legacy) { + for vector in load_fixture() + .vectors + .into_iter() + .filter(|v| !v.is_legacy()) + { let input = hex::decode(&vector.input_hex).expect("input_hex must decode"); let expected = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); @@ -193,12 +197,16 @@ fn vectors_reencode_byte_identical() { /// codec test failing too means the MessagePack layout changed (protocol#11 /// territory); codec test passing means the LZ4 block encoding changed. /// -/// Legacy vectors only, same reason as `vectors_reencode_byte_identical`: -/// re-serialization emits the writer's current encoding, so byte-identity can -/// only hold for the set that matches it. +/// `*_bin` vectors only, same reason as `vectors_reencode_byte_identical`: +/// re-serialization emits the writer's current encoding (`bin`), so +/// byte-identity can only hold for the set that matches it. #[test] fn envelope_codec_roundtrip_byte_identical() { - for vector in load_fixture().vectors.into_iter().filter(Vector::is_legacy) { + for vector in load_fixture() + .vectors + .into_iter() + .filter(|v| !v.is_legacy()) + { let canonical = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); let envelope: StorageEnvelope = rmp_serde::from_slice(&canonical) .unwrap_or_else(|e| panic!("[{}] envelope must deserialize: {e}", vector.name));