From 0f9334ad5be476b8af1694c5da2cc9243646a853 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sat, 25 Jul 2026 16:20:03 +1000 Subject: [PATCH 1/3] test: prove dual-decode of legacy + bin envelopes, re-pin protocol 1.1 fixture (LAB-791) Readers-first release for the StorageEnvelope compressed_data bin encoding (protocol 1.1, decisions/envelope-bin-encoding.md; rollout step 1 of #54): - Re-vendor tests/vectors/wire-format.json from protocol@6863495d (fixture 1.1.0: 6 legacy vectors + 6 *_bin twins) and re-pin FIXTURE_SHA256 with updated provenance. - Decode-identity through ByteStorage::retrieve() now runs against BOTH vector sets, forever; re-encode byte-identity stays pinned to the legacy set until the writer flip (which switches it to *_bin). - tests/dual_decode.rs: full reader matrix (plain Vec reader and serde_bytes reader x array/bin wire) over every pinned vector, seeded from the LAB-764 toolchain experiment, plus bin8/bin16 boundary sweep and bin32 round-trips with real LZ4 output. No src/ change; the writer still emits legacy array-of-ints. Test-only patch release so the stage-3 SDK rebuilds have a version to pin. Release-As: 0.3.1 --- README.md | 16 +- tests/dual_decode.rs | 267 +++++++++++++++++++++++++++++++++ tests/vectors/wire-format.json | 72 ++++++++- tests/wire_format_vectors.rs | 135 +++++++++++++++-- 4 files changed, 468 insertions(+), 22 deletions(-) create mode 100644 tests/dual_decode.rs diff --git a/README.md b/README.md index 6161eef..a218f84 100644 --- a/README.md +++ b/README.md @@ -336,11 +336,17 @@ See [`fuzz/README.md`](fuzz/README.md) for comprehensive fuzzing documentation. `tests/wire_format_vectors.rs` byte-verifies this crate against the canonical ByteStorage envelope vectors from [`cachekit-io/protocol`](https://github.com/cachekit-io/protocol) -(`test-vectors/wire-format.json`): every vector must decode to the exact -payload bytes and re-encode to the exact envelope bytes. The fixture is -vendored at `tests/vectors/wire-format.json` and integrity-pinned by sha256 — -to update it, re-copy from the protocol repo and change the pinned hash in the -same commit. +(`test-vectors/wire-format.json`). Since protocol 1.1 the fixture pins two +`compressed_data` encodings — legacy array-of-integers vectors and their +`*_bin` twins (msgpack `bin`, canonical for 1.1+ writers): every vector in +both sets must decode to the exact payload bytes, while re-encode +byte-identity is asserted against the set matching this crate's current +writer encoding (legacy, until the `serde_bytes` writer flip lands). +`tests/dual_decode.rs` proves both reader shapes accept both encodings, +including bin16/bin32 width headers. The fixture is vendored at +`tests/vectors/wire-format.json` and integrity-pinned by sha256 — to update +it, re-copy from the protocol repo and change the pinned hash in the same +commit. --- diff --git a/tests/dual_decode.rs b/tests/dual_decode.rs new file mode 100644 index 0000000..d35ea82 --- /dev/null +++ b/tests/dual_decode.rs @@ -0,0 +1,267 @@ +//! Dual-decode proof for the StorageEnvelope `compressed_data` bin encoding. +//! +//! Protocol 1.1 (`protocol/decisions/envelope-bin-encoding.md`) flips +//! `compressed_data` from the legacy msgpack array-of-integers encoding to +//! msgpack `bin` — readers first. This file is the readers-first CI proof: +//! under the locked decoder stack (rmp-serde + serde_bytes), BOTH reader +//! shapes accept BOTH encodings, so `bin` envelopes written by 1.1+ writers +//! are readable today and legacy envelopes stay readable forever. +//! +//! The full matrix, executed against the byte-pinned protocol vectors: +//! +//! | Reader | legacy wire (array) | new wire (bin) | +//! |------------------------------------|---------------------|----------------| +//! | old (plain `Vec`, shipped) | status quo | proven here | +//! | …incl. `ByteStorage::retrieve()` | | proven here | +//! | new (`serde_bytes`, writer-flip) | proven here | proven here | +//! +//! Seeded from the LAB-764 toolchain experiment (`dual_decode_experiment.rs`, +//! attached to the decision memo on that issue); the throughput half of the +//! experiment is deliberately absent — benchmark evidence belongs to the +//! writer-flip stage (LAB-510 harness), not this test-only release. +//! +//! Width boundaries: the fixture's `*_bin` twins all fit in a bin8 header +//! (`0xc4`, ≤ 255 B). The `width_boundary_*` tests cover the bin16 (`0xc5`) +//! and bin32 (`0xc6`) headers with real LZ4 output, per the MUST recorded in +//! the decision record — fixture-level width vectors land with the writer +//! flip, where the real writer can generate them natively. + +#![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] + +use cachekit_core::{ByteStorage, StorageEnvelope}; +use serde::{Deserialize, Serialize}; + +const FIXTURE: &str = include_str!("vectors/wire-format.json"); + +/// Post-writer-flip twin of StorageEnvelope: identical shape, `serde_bytes` +/// on `compressed_data` only. `checksum: [u8; 8]` deliberately stays +/// array-of-ints — normative scope exclusion in the decision record +/// (crypto-adjacent surface; implementations MUST NOT flip it +/// opportunistically). +#[derive(Serialize, Deserialize, PartialEq, Debug)] +struct StorageEnvelopeV2 { + #[serde(with = "serde_bytes")] + compressed_data: Vec, + checksum: [u8; 8], + original_size: u32, + format: String, +} + +#[derive(serde::Deserialize)] +struct WireFormatFixture { + vectors: Vec, +} + +#[derive(serde::Deserialize)] +struct Vector { + name: String, + input_hex: String, + envelope_hex: String, + envelope_encoding: Option, +} + +fn load_vectors() -> Vec { + serde_json::from_str::(FIXTURE) + .expect("fixture parses") + .vectors +} + +/// Assert one envelope's bytes decode identically through every reader shape: +/// old struct, new struct, and the full shipped `retrieve()` path (checksum +/// validation + decompression-ratio guard included). +fn assert_all_readers_decode(name: &str, wire: &[u8], expected_payload: &[u8]) { + let old: StorageEnvelope = rmp_serde::from_slice(wire) + .unwrap_or_else(|e| panic!("[{name}] old reader (plain Vec) failed: {e}")); + let new: StorageEnvelopeV2 = rmp_serde::from_slice(wire) + .unwrap_or_else(|e| panic!("[{name}] new reader (serde_bytes) failed: {e}")); + assert_eq!(old.compressed_data, new.compressed_data, "[{name}]"); + assert_eq!(old.checksum, new.checksum, "[{name}]"); + assert_eq!(old.original_size, new.original_size, "[{name}]"); + assert_eq!(old.format, new.format, "[{name}]"); + + let storage = ByteStorage::new(None); + let (payload, _format) = storage + .retrieve(wire) + .unwrap_or_else(|e| panic!("[{name}] shipped retrieve() rejected envelope: {e:?}")); + assert_eq!( + payload, expected_payload, + "[{name}] payload mismatch via retrieve()" + ); +} + +/// The 4-cell matrix over the legacy pinned vectors: decode each with both +/// reader shapes, re-encode through the new (serde_bytes) shape to produce +/// bin wire, and prove that bin wire decodes through both reader shapes AND +/// today's shipped `retrieve()`. +#[test] +fn dual_decode_matrix_against_legacy_vectors() { + let vectors = load_vectors(); + let legacy: Vec<&Vector> = vectors + .iter() + .filter(|v| v.envelope_encoding.is_none()) + .collect(); + assert!(legacy.len() >= 6, "expected the pinned legacy vector set"); + + for v in legacy { + let old_wire = hex::decode(&v.envelope_hex).unwrap(); + let input = hex::decode(&v.input_hex).unwrap(); + + // Cells: old reader <- old wire (baseline), new reader <- old wire + // (the readers-first claim), plus retrieve() on the pinned bytes. + assert_all_readers_decode(&v.name, &old_wire, &input); + + // Produce new wire the way the flipped writer will. + let new_from_old: StorageEnvelopeV2 = rmp_serde::from_slice(&old_wire).unwrap(); + let new_wire = rmp_serde::to_vec(&new_from_old).unwrap(); + + // compressed_data (element [0], right after the 0x94 fixarray marker) + // must now carry a bin marker. + assert_eq!( + new_wire[0], 0x94, + "[{}] outer fixarray(4) preserved", + v.name + ); + assert!( + // 0xc4/0xc5/0xc6 = msgpack bin8/bin16/bin32 + matches!(new_wire[1], 0xc4..=0xc6), + "[{}] compressed_data not bin-encoded: 0x{:02x}", + v.name, + new_wire[1] + ); + // Documented micro-regression: bin loses at most a header byte or two + // on tiny envelopes (bin8 header 2 B vs fixarray 1 B); every payload + // byte >= 0x80 saves a byte. Growth is bounded by header slack. + assert!( + new_wire.len() <= old_wire.len() + 4, + "[{}] new wire larger than old beyond header slack", + v.name + ); + + // Cells: old reader <- new wire (rollout-order bonus), new reader <- + // new wire (round-trip), retrieve() <- new wire (API-level proof). + assert_all_readers_decode(&v.name, &new_wire, &input); + } +} + +/// The pinned `*_bin` twins through the same matrix, plus the writer-flip +/// forecast: re-encoding a twin through the serde_bytes shape must be +/// byte-identical to the protocol's pinned bin bytes — the writer-flip diff +/// will point `vectors_reencode_byte_identical` at this set, and this assert +/// guarantees that switch lands green. +#[test] +fn dual_decode_matrix_against_bin_vectors() { + let vectors = load_vectors(); + let bin: Vec<&Vector> = vectors + .iter() + .filter(|v| v.envelope_encoding.is_some()) + .collect(); + assert!(bin.len() >= 6, "expected the pinned *_bin twin set"); + + for v in bin { + let wire = hex::decode(&v.envelope_hex).unwrap(); + let input = hex::decode(&v.input_hex).unwrap(); + + assert_all_readers_decode(&v.name, &wire, &input); + + let env: StorageEnvelopeV2 = rmp_serde::from_slice(&wire).unwrap(); + let reencoded = rmp_serde::to_vec(&env).unwrap(); + assert_eq!( + hex::encode(&reencoded), + hex::encode(&wire), + "[{}] serde_bytes re-encode is not byte-identical to the pinned bin vector", + v.name + ); + } +} + +/// Deterministic xorshift64* stream — incompressible payload without a rand +/// dependency (same generator as the LAB-764 experiment). +fn incompressible(len: usize) -> Vec { + let mut state: u64 = 0x9e3779b97f4a7c15; + let mut out = Vec::with_capacity(len + 8); + while out.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + out.extend_from_slice(&state.wrapping_mul(0x2545f4914f6cdd1d).to_le_bytes()); + } + out.truncate(len); + out +} + +/// Build a real envelope (real LZ4 output) from an incompressible payload, +/// bin-encode it via the serde_bytes shape, and assert the expected bin width +/// marker before running the full reader matrix on it. +fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) -> usize { + let payload = incompressible(payload_len); + let env = StorageEnvelope::new(&payload, "msgpack".to_string()).unwrap(); + let compressed_len = env.compressed_data.len(); + + let v2 = StorageEnvelopeV2 { + compressed_data: env.compressed_data, + checksum: env.checksum, + original_size: env.original_size, + format: env.format, + }; + let wire = rmp_serde::to_vec(&v2).unwrap(); + assert_eq!(wire[0], 0x94, "[{name}] outer fixarray(4) preserved"); + assert_eq!( + wire[1], expected_marker, + "[{name}] expected bin marker 0x{expected_marker:02x} for compressed_data of {compressed_len} B, got 0x{:02x}", + wire[1] + ); + + assert_all_readers_decode(name, &wire, &payload); + compressed_len +} + +/// bin8/bin16 boundary: the fixture twins never leave bin8 territory (all +/// compressed_data <= 255 B), so sweep real LZ4 outputs across the 255/256 B +/// boundary and prove both header widths decode through every reader shape. +/// LZ4 adds a few bytes of literal-run overhead on incompressible input, so +/// the sweep brackets the boundary regardless of the exact overhead. +#[test] +fn width_boundary_bin8_bin16() { + let mut seen_bin8 = false; + let mut seen_bin16 = false; + for payload_len in 230..=270 { + let payload = incompressible(payload_len); + let env = StorageEnvelope::new(&payload, "msgpack".to_string()).unwrap(); + let expected = if env.compressed_data.len() <= 255 { + seen_bin8 = true; + 0xc4 // bin8 + } else { + seen_bin16 = true; + 0xc5 // bin16 + }; + assert_width_tier( + &format!("bin8/bin16 sweep len={payload_len}"), + payload_len, + expected, + ); + } + assert!( + seen_bin8 && seen_bin16, + "sweep must land on both sides of the 255 B boundary (bin8 seen: {seen_bin8}, bin16 seen: {seen_bin16})" + ); +} + +/// bin16/bin32 boundary: compressed_data > 65,535 B forces the bin32 header. +/// One representative envelope per side, full reader matrix on each. +#[test] +fn width_boundary_bin16_bin32() { + // Comfortably inside bin16 (compressed ~4 KiB). + let len16 = assert_width_tier("bin16 tier", 4 * 1024, 0xc5); + assert!( + (256..=65535).contains(&len16), + "bin16 tier compressed_data landed outside (256..=65535): {len16} B" + ); + + // Incompressible 68 KiB compresses to > 65,535 B (LZ4 overhead is + // positive on incompressible input), forcing bin32. + let len32 = assert_width_tier("bin32 tier", 68 * 1024, 0xc6); + assert!( + len32 > 65535, + "bin32 tier compressed_data must exceed 65535 B: {len32} B" + ); +} diff --git a/tests/vectors/wire-format.json b/tests/vectors/wire-format.json index dde964c..9be2ba0 100644 --- a/tests/vectors/wire-format.json +++ b/tests/vectors/wire-format.json @@ -1,8 +1,8 @@ { "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input", "compression": "LZ4 block format (NOT frame)", - "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]; byte fields encode as msgpack arrays of integers, not bin", - "generator": "cachekit-core v0.2.0", + "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]. Vectors without an 'envelope_encoding' field use the legacy array-of-integers encoding for compressed_data and are retained forever as legacy-read proof; their '*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). checksum always encodes as an array of 8 integers. Normative rules: spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'.", + "generator": "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)", "limits": { "max_compressed_size": 536870912, "max_compression_ratio": 1000, @@ -62,7 +62,73 @@ "input_hex": "ff", "input_size": 1, "name": "single_byte" + }, + { + "description": "bin-encoded twin of 'empty': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40100982d06cc800538ccd3cc94ccc200a76d73677061636b", + "envelope_size": 26, + "format": "msgpack", + "input_hex": "", + "input_size": 0, + "name": "empty_bin", + "envelope_encoding": "bin", + "derived_from": "empty" + }, + { + "description": "bin-encoded twin of 'simple_string': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c412f00168656c6c6f2c2063616368656b697421986dcc8a34ccb63a3c52ccd310a76d73677061636b", + "envelope_size": 42, + "format": "msgpack", + "input_hex": "68656c6c6f2c2063616368656b697421", + "input_size": 16, + "name": "simple_string_bin", + "envelope_encoding": "bin", + "derived_from": "simple_string" + }, + { + "description": "bin-encoded twin of 'binary_data': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c442f031243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917982d016e0411ccedcc9e3440a76d73677061636b", + "envelope_size": 89, + "format": "msgpack", + "input_hex": "243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917", + "input_size": 64, + "name": "binary_data_bin", + "envelope_encoding": "bin", + "derived_from": "binary_data" + }, + { + "description": "bin-encoded twin of 'large_compressible': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40f1f410100ffffffe960414141414141984d35ccad08cce9ccd77c0dcd0400a76d73677061636b", + "envelope_size": 41, + "format": "msgpack", + "input_hex": "41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141", + "input_size": 1024, + "name": "large_compressible_bin", + "envelope_encoding": "bin", + "derived_from": "large_compressible" + }, + { + "description": "bin-encoded twin of 'json_like': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c44cf03b7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d98cc87673661cc9c4bcca8204aa76d73677061636b", + "envelope_size": 100, + "format": "msgpack", + "input_hex": "7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d", + "input_size": 74, + "name": "json_like_bin", + "envelope_encoding": "bin", + "derived_from": "json_like" + }, + { + "description": "bin-encoded twin of 'single_byte': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40210ff98ccd6ccbcccec3c6b29ccd72e01a76d73677061636b", + "envelope_size": 27, + "format": "msgpack", + "input_hex": "ff", + "input_size": 1, + "name": "single_byte_bin", + "envelope_encoding": "bin", + "derived_from": "single_byte" } ], - "version": "1.0.0" + "version": "1.1.0" } diff --git a/tests/wire_format_vectors.rs b/tests/wire_format_vectors.rs index c4c5297..9fb2142 100644 --- a/tests/wire_format_vectors.rs +++ b/tests/wire_format_vectors.rs @@ -3,14 +3,21 @@ //! Verifies this crate — the canonical ByteStorage implementation — against the //! byte-canonical envelope vectors pinned by the protocol spec //! (`protocol/spec/wire-format.md`). Every vector must decode to the expected -//! payload bytes AND re-encode to the exact envelope bytes, so any encoding -//! regression (e.g. the positional-fixarray / int-array facts corrected in -//! protocol#11) fails CI here instead of breaking cross-version reads in -//! production. +//! payload bytes, so any decoding regression fails CI here instead of breaking +//! cross-version reads in production. +//! +//! Since protocol 1.1 (`decisions/envelope-bin-encoding.md`) the fixture pins +//! two encodings of `compressed_data`: legacy array-of-integers vectors +//! (no `envelope_encoding` field, retained forever as legacy-read proof) and +//! their `*_bin` twins (`"envelope_encoding": "bin"`, canonical for 1.1+ +//! writers). Decode-identity runs against BOTH sets, forever. Re-encode +//! byte-identity runs against the set matching this crate's CURRENT writer +//! encoding — legacy today; the writer-flip diff (`serde_bytes` on +//! `compressed_data`) switches those assertions to the `*_bin` set. //! //! Fixture provenance: vendored from //! `test-vectors/wire-format.json` -//! at commit `b0270b999bf827f8aa5fcc2d8640735196510326`, integrity-pinned by +//! at commit `6863495d3565ccd9f28827baa1351782575e770d`, integrity-pinned by //! sha256 below. To update: copy the file from a newer protocol ref, update //! `FIXTURE_SHA256` and this comment's commit hash together. @@ -24,7 +31,7 @@ use sha2::{Digest, Sha256}; const FIXTURE: &str = include_str!("vectors/wire-format.json"); /// sha256 of the vendored fixture — must match the protocol repo's copy. -const FIXTURE_SHA256: &str = "f83a8a18c5067c6353b7c13f0a47b4e5cefec11f4ab881b12079483174e6e0b8"; +const FIXTURE_SHA256: &str = "d6184a945d8cb22491b5c937414fe4d01e682d084ccec9dae9526d0ffba650cb"; #[derive(serde::Deserialize)] struct WireFormatFixture { @@ -48,6 +55,19 @@ struct Vector { input_size: usize, envelope_hex: String, envelope_size: usize, + /// `None` = legacy array-of-integers encoding; `Some("bin")` = msgpack bin + /// (canonical for protocol 1.1+ writers). + envelope_encoding: Option, + /// For `*_bin` twins: the legacy vector they were derived from. + derived_from: Option, +} + +impl Vector { + /// Encoded with the legacy array-of-integers `compressed_data` — the + /// encoding this crate's writer currently emits. + fn is_legacy(&self) -> bool { + self.envelope_encoding.is_none() + } } fn load_fixture() -> WireFormatFixture { @@ -67,11 +87,16 @@ fn fixture_integrity_pinned_sha256() { #[test] fn fixture_is_current_version_with_vectors() { let fixture = load_fixture(); - assert_eq!(fixture.version, "1.0.0"); + assert_eq!(fixture.version, "1.1.0"); + let legacy = fixture.vectors.iter().filter(|v| v.is_legacy()).count(); + let bin = fixture.vectors.len() - legacy; assert!( - fixture.vectors.len() >= 6, - "vector set is append-only; expected at least the original 6, got {}", - fixture.vectors.len() + legacy >= 6, + "legacy vectors are retained forever as legacy-read proof; expected at least the original 6, got {legacy}" + ); + assert!( + bin >= 6, + "protocol 1.1 pins a *_bin twin per legacy vector; expected at least 6, got {bin}" ); } @@ -93,8 +118,11 @@ fn fixture_limits_match_implementation() { ); } -/// Decode direction: every canonical envelope must retrieve to the exact -/// original payload bytes, format, and size. +/// Decode direction: every canonical envelope — BOTH legacy array-of-integers +/// vectors and their `*_bin` twins — must retrieve to the exact original +/// payload bytes, format, and size. This set never shrinks: legacy vectors are +/// legacy-read proof, bin vectors prove readers accept the 1.1+ canonical +/// writer encoding before any writer emits it (readers-first rollout). #[test] fn vectors_decode_to_expected_payload() { let storage = ByteStorage::new(None); @@ -136,10 +164,14 @@ fn vectors_decode_to_expected_payload() { /// MessagePack) is deterministic; a failure here means the wire bytes changed /// and cross-version reads are at risk — regenerate vectors deliberately in /// the protocol repo, never adjust expectations here. +/// +/// Legacy vectors only: this asserts against the encoding the writer CURRENTLY +/// emits. The writer-flip diff switches this filter to the `*_bin` set in the +/// same PR that adds `serde_bytes` to `compressed_data`. #[test] fn vectors_reencode_byte_identical() { let storage = ByteStorage::new(None); - for vector in load_fixture().vectors { + for vector in load_fixture().vectors.into_iter().filter(Vector::is_legacy) { let input = hex::decode(&vector.input_hex).expect("input_hex must decode"); let expected = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); @@ -160,9 +192,13 @@ fn vectors_reencode_byte_identical() { /// `vectors_reencode_byte_identical` fails, this localizes the regression — /// codec test failing too means the MessagePack layout changed (protocol#11 /// territory); codec test passing means the LZ4 block encoding changed. +/// +/// Legacy vectors only, same reason as `vectors_reencode_byte_identical`: +/// re-serialization emits the writer's current encoding, so byte-identity can +/// only hold for the set that matches it. #[test] fn envelope_codec_roundtrip_byte_identical() { - for vector in load_fixture().vectors { + for vector in load_fixture().vectors.into_iter().filter(Vector::is_legacy) { let canonical = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); let envelope: StorageEnvelope = rmp_serde::from_slice(&canonical) .unwrap_or_else(|e| panic!("[{}] envelope must deserialize: {e}", vector.name)); @@ -176,3 +212,74 @@ fn envelope_codec_roundtrip_byte_identical() { ); } } + +/// Every `*_bin` twin must carry an actual msgpack bin marker on +/// `compressed_data` (element [0], right after the 0x94 fixarray header) and +/// deserialize to a StorageEnvelope field-identical to its legacy parent — +/// the fixture's "identical fields, different encoding" contract. Guards +/// against a regenerated fixture silently shipping twins that don't actually +/// exercise the bin decode path. +#[test] +fn bin_twins_are_bin_encoded_and_field_identical_to_legacy() { + let fixture = load_fixture(); + let twins: Vec<&Vector> = fixture.vectors.iter().filter(|v| !v.is_legacy()).collect(); + assert!( + !twins.is_empty(), + "protocol 1.1 fixture must carry *_bin twins" + ); + + for twin in twins { + assert_eq!( + twin.envelope_encoding.as_deref(), + Some("bin"), + "[{}] unknown envelope_encoding", + twin.name + ); + let parent_name = twin + .derived_from + .as_deref() + .unwrap_or_else(|| panic!("[{}] *_bin twin missing derived_from", twin.name)); + let parent = fixture + .vectors + .iter() + .find(|v| v.name == parent_name) + .unwrap_or_else(|| { + panic!( + "[{}] derived_from '{parent_name}' not in fixture", + twin.name + ) + }); + + let twin_bytes = hex::decode(&twin.envelope_hex).expect("envelope_hex must decode"); + assert_eq!( + twin_bytes[0], 0x94, + "[{}] outer fixarray(4) marker", + twin.name + ); + assert!( + // 0xc4/0xc5/0xc6 = msgpack bin8/bin16/bin32 + matches!(twin_bytes[1], 0xc4..=0xc6), + "[{}] compressed_data is not bin-encoded: marker 0x{:02x}", + twin.name, + twin_bytes[1] + ); + + let parent_bytes = hex::decode(&parent.envelope_hex).expect("envelope_hex must decode"); + let twin_env: StorageEnvelope = rmp_serde::from_slice(&twin_bytes) + .unwrap_or_else(|e| panic!("[{}] twin must deserialize: {e}", twin.name)); + let parent_env: StorageEnvelope = rmp_serde::from_slice(&parent_bytes) + .unwrap_or_else(|e| panic!("[{parent_name}] parent must deserialize: {e}")); + assert_eq!( + twin_env.compressed_data, parent_env.compressed_data, + "[{}]", + twin.name + ); + assert_eq!(twin_env.checksum, parent_env.checksum, "[{}]", twin.name); + assert_eq!( + twin_env.original_size, parent_env.original_size, + "[{}]", + twin.name + ); + assert_eq!(twin_env.format, parent_env.format, "[{}]", twin.name); + } +} From ac7bf7f6401810231bce0965c24539cc045e1ce8 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sat, 25 Jul 2026 16:33:50 +1000 Subject: [PATCH 2/3] test: add vector-name context to unwraps in dual_decode (LAB-791) CodeRabbit review: bare .unwrap() calls lost which-vector context on failure; align with wire_format_vectors.rs's panic-with-name convention. --- tests/dual_decode.rs | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/dual_decode.rs b/tests/dual_decode.rs index d35ea82..f414a3f 100644 --- a/tests/dual_decode.rs +++ b/tests/dual_decode.rs @@ -103,16 +103,20 @@ fn dual_decode_matrix_against_legacy_vectors() { assert!(legacy.len() >= 6, "expected the pinned legacy vector set"); for v in legacy { - let old_wire = hex::decode(&v.envelope_hex).unwrap(); - let input = hex::decode(&v.input_hex).unwrap(); + let old_wire = hex::decode(&v.envelope_hex) + .unwrap_or_else(|e| panic!("[{}] envelope_hex must decode: {e}", v.name)); + let input = hex::decode(&v.input_hex) + .unwrap_or_else(|e| panic!("[{}] input_hex must decode: {e}", v.name)); // Cells: old reader <- old wire (baseline), new reader <- old wire // (the readers-first claim), plus retrieve() on the pinned bytes. assert_all_readers_decode(&v.name, &old_wire, &input); // Produce new wire the way the flipped writer will. - let new_from_old: StorageEnvelopeV2 = rmp_serde::from_slice(&old_wire).unwrap(); - let new_wire = rmp_serde::to_vec(&new_from_old).unwrap(); + let new_from_old: StorageEnvelopeV2 = rmp_serde::from_slice(&old_wire) + .unwrap_or_else(|e| panic!("[{}] new reader must decode legacy wire: {e}", v.name)); + let new_wire = rmp_serde::to_vec(&new_from_old) + .unwrap_or_else(|e| panic!("[{}] serde_bytes shape must serialize: {e}", v.name)); // compressed_data (element [0], right after the 0x94 fixarray marker) // must now carry a bin marker. @@ -158,13 +162,17 @@ fn dual_decode_matrix_against_bin_vectors() { assert!(bin.len() >= 6, "expected the pinned *_bin twin set"); for v in bin { - let wire = hex::decode(&v.envelope_hex).unwrap(); - let input = hex::decode(&v.input_hex).unwrap(); + let wire = hex::decode(&v.envelope_hex) + .unwrap_or_else(|e| panic!("[{}] envelope_hex must decode: {e}", v.name)); + let input = hex::decode(&v.input_hex) + .unwrap_or_else(|e| panic!("[{}] input_hex must decode: {e}", v.name)); assert_all_readers_decode(&v.name, &wire, &input); - let env: StorageEnvelopeV2 = rmp_serde::from_slice(&wire).unwrap(); - let reencoded = rmp_serde::to_vec(&env).unwrap(); + let env: StorageEnvelopeV2 = rmp_serde::from_slice(&wire) + .unwrap_or_else(|e| panic!("[{}] new reader must decode bin wire: {e}", v.name)); + let reencoded = rmp_serde::to_vec(&env) + .unwrap_or_else(|e| panic!("[{}] serde_bytes shape must serialize: {e}", v.name)); assert_eq!( hex::encode(&reencoded), hex::encode(&wire), @@ -194,7 +202,8 @@ fn incompressible(len: usize) -> Vec { /// marker before running the full reader matrix on it. fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) -> usize { let payload = incompressible(payload_len); - let env = StorageEnvelope::new(&payload, "msgpack".to_string()).unwrap(); + let env = StorageEnvelope::new(&payload, "msgpack".to_string()) + .unwrap_or_else(|e| panic!("[{name}] envelope construction failed: {e:?}")); let compressed_len = env.compressed_data.len(); let v2 = StorageEnvelopeV2 { @@ -203,7 +212,8 @@ fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) -> usi original_size: env.original_size, format: env.format, }; - let wire = rmp_serde::to_vec(&v2).unwrap(); + let wire = rmp_serde::to_vec(&v2) + .unwrap_or_else(|e| panic!("[{name}] serde_bytes shape must serialize: {e}")); assert_eq!(wire[0], 0x94, "[{name}] outer fixarray(4) preserved"); assert_eq!( wire[1], expected_marker, @@ -226,7 +236,9 @@ fn width_boundary_bin8_bin16() { let mut seen_bin16 = false; for payload_len in 230..=270 { let payload = incompressible(payload_len); - let env = StorageEnvelope::new(&payload, "msgpack".to_string()).unwrap(); + let env = StorageEnvelope::new(&payload, "msgpack".to_string()).unwrap_or_else(|e| { + panic!("[sweep len={payload_len}] envelope construction failed: {e:?}") + }); let expected = if env.compressed_data.len() <= 255 { seen_bin8 = true; 0xc4 // bin8 From e4ae645030f8cdae721978fbf7829527fc98aa50 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 26 Jul 2026 04:13:11 +1000 Subject: [PATCH 3/3] test: tighten dual-decode drift guards per expert panel (LAB-791) Apply the FIX-FIRST expert-panel findings on cachekit-core#57 (all test-only, no src/ change, wire bytes unchanged): - must-fix: tighten the bin micro-regression bound from +4 to +1 B. The panel re-derived and measured (lz4_flex 0.12.2) worst-case delta(bin-legacy) as exactly +1 ("empty", 25->26 B); at +4 a +2/+3 bloat regression stays green -- the exact false-green this readers-first drift gate exists to catch, and the ceiling the writer-flip step (LAB-764) will read as the micro-regression contract. - should-fix: pin the fixed-size bin marker asserts to == 0xc4 (bin8) in dual_decode.rs and wire_format_vectors.rs. The 0xc4..=0xc6 range accepted any width, so a width-selection regression emitting bin16/bin32 on a tiny payload passed while breaking cross-SDK byte-identity. Range kept only in the width_boundary_* tests that legitimately span widths. - cut: drop the redundant (256..=65535)/>65535 range asserts in width_boundary_bin16_bin32 (dominated by the 0xc5/0xc6 marker asserts -- rmp selects width by length, so marker <=> range) and assert_width_tier's now-unused -> usize return. Optional cosmetic finding (per-vector [name] context on fixture hex-decode expects) deferred: the sha256-pinned fixture already makes any decode failure a whole-suite RED, so per-site context adds little for 7 sites of churn. fmt + clippy -D warnings + affected integration tests all green. Co-authored-by: multica-agent --- tests/dual_decode.rs | 45 +++++++++++++++++++----------------- tests/wire_format_vectors.rs | 10 ++++---- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/tests/dual_decode.rs b/tests/dual_decode.rs index f414a3f..862dcc4 100644 --- a/tests/dual_decode.rs +++ b/tests/dual_decode.rs @@ -125,19 +125,27 @@ fn dual_decode_matrix_against_legacy_vectors() { "[{}] outer fixarray(4) preserved", v.name ); - assert!( - // 0xc4/0xc5/0xc6 = msgpack bin8/bin16/bin32 - matches!(new_wire[1], 0xc4..=0xc6), - "[{}] compressed_data not bin-encoded: 0x{:02x}", + assert_eq!( + // 0xc4 = msgpack bin8. Every pinned twin's compressed_data is small + // enough to be bin8; a wider marker here is a width-selection + // regression that would break cross-SDK byte-identity. + new_wire[1], + 0xc4, + "[{}] compressed_data not bin8-encoded: 0x{:02x}", v.name, new_wire[1] ); - // Documented micro-regression: bin loses at most a header byte or two - // on tiny envelopes (bin8 header 2 B vs fixarray 1 B); every payload - // byte >= 0x80 saves a byte. Growth is bounded by header slack. + // Documented micro-regression: bin costs at most +1 B, and only on the + // smallest envelopes. A <=15 B compressed_data goes from a 1 B fixarray + // header to a 2 B bin8 header (+1); any array >=16 B already carried a + // 3 B array16 header, so bin8's 2 B header shrinks it. Measured worst + // case on the pinned set is exactly +1 (`empty`, 25 -> 26 B); every + // other vector ties or shrinks. This +1 B ceiling is the micro- + // regression contract the writer-flip step (LAB-764) will read, so keep + // it tight — a +2/+3 bloat regression must not slip through green. assert!( - new_wire.len() <= old_wire.len() + 4, - "[{}] new wire larger than old beyond header slack", + new_wire.len() <= old_wire.len() + 1, + "[{}] new wire exceeds the +1 B bin8-header ceiling", v.name ); @@ -200,7 +208,7 @@ fn incompressible(len: usize) -> Vec { /// Build a real envelope (real LZ4 output) from an incompressible payload, /// bin-encode it via the serde_bytes shape, and assert the expected bin width /// marker before running the full reader matrix on it. -fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) -> usize { +fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) { let payload = incompressible(payload_len); let env = StorageEnvelope::new(&payload, "msgpack".to_string()) .unwrap_or_else(|e| panic!("[{name}] envelope construction failed: {e:?}")); @@ -222,7 +230,6 @@ fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) -> usi ); assert_all_readers_decode(name, &wire, &payload); - compressed_len } /// bin8/bin16 boundary: the fixture twins never leave bin8 territory (all @@ -262,18 +269,14 @@ fn width_boundary_bin8_bin16() { /// One representative envelope per side, full reader matrix on each. #[test] fn width_boundary_bin16_bin32() { + // The 0xc5 / 0xc6 marker assert inside assert_width_tier already proves the + // tier: rmp selects the bin width purely by length, so marker <=> range. + // A separate range assert on the returned length would only restate that. + // Comfortably inside bin16 (compressed ~4 KiB). - let len16 = assert_width_tier("bin16 tier", 4 * 1024, 0xc5); - assert!( - (256..=65535).contains(&len16), - "bin16 tier compressed_data landed outside (256..=65535): {len16} B" - ); + assert_width_tier("bin16 tier", 4 * 1024, 0xc5); // Incompressible 68 KiB compresses to > 65,535 B (LZ4 overhead is // positive on incompressible input), forcing bin32. - let len32 = assert_width_tier("bin32 tier", 68 * 1024, 0xc6); - assert!( - len32 > 65535, - "bin32 tier compressed_data must exceed 65535 B: {len32} B" - ); + assert_width_tier("bin32 tier", 68 * 1024, 0xc6); } diff --git a/tests/wire_format_vectors.rs b/tests/wire_format_vectors.rs index 9fb2142..1eae4be 100644 --- a/tests/wire_format_vectors.rs +++ b/tests/wire_format_vectors.rs @@ -256,10 +256,12 @@ fn bin_twins_are_bin_encoded_and_field_identical_to_legacy() { "[{}] outer fixarray(4) marker", twin.name ); - assert!( - // 0xc4/0xc5/0xc6 = msgpack bin8/bin16/bin32 - matches!(twin_bytes[1], 0xc4..=0xc6), - "[{}] compressed_data is not bin-encoded: marker 0x{:02x}", + assert_eq!( + // 0xc4 = msgpack bin8. Every pinned twin is small enough to be + // bin8; a wider marker here is a width-selection regression. + twin_bytes[1], + 0xc4, + "[{}] compressed_data is not bin8-encoded: marker 0x{:02x}", twin.name, twin_bytes[1] );