feat: emit StorageEnvelope compressed_data as msgpack bin (protocol 1.1) (LAB-866)

README.md

@@ -341,7 +341,10 @@ ByteStorage envelope vectors from
 `*_bin` twins (msgpack `bin`, canonical for 1.1+ writers): every vector in
 both sets must decode to the exact payload bytes, while re-encode
 byte-identity is asserted against the set matching this crate's current
-writer encoding (legacy, until the `serde_bytes` writer flip lands).
+writer encoding — msgpack `bin`, since the protocol 1.1 `serde_bytes`
+writer flip (LAB-866; `checksum` deliberately stays array-of-ints per the
+protocol's normative scope exclusion). Legacy envelopes remain readable
+forever.
 `tests/dual_decode.rs` proves both reader shapes accept both encodings,
 including bin16/bin32 width headers. The fixture is vendored at
 `tests/vectors/wire-format.json` and integrity-pinned by sha256 — to update

benches/hot_path.rs

@@ -7,7 +7,7 @@
 //! paths are identified. Sizes chosen to span the realistic cache-payload
 //! distribution (64B keys, 1KB values, 64KB large objects).
 
-use cachekit_core::{ByteStorage, ZeroKnowledgeEncryptor};
+use cachekit_core::{ByteStorage, StorageEnvelope, ZeroKnowledgeEncryptor};
 use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
 
 const SIZES: &[usize] = &[64, 256, 1024, 4 * 1024, 16 * 1024, 64 * 1024];
@@ -16,6 +16,64 @@ fn make_payload(size: usize) -> Vec<u8> {
     (0..size).map(|i| (i % 256) as u8).collect()
 }
 
+/// Deterministic xorshift64* stream — incompressible payload without a rand
+/// dependency (same generator as tests/dual_decode.rs).
+fn incompressible(len: usize) -> Vec<u8> {
+    let mut state: u64 = 0x9e3779b97f4a7c15;
+    let mut out = Vec::with_capacity(len + 8);
+    while out.len() < len {
+        state ^= state << 13;
+        state ^= state >> 7;
+        state ^= state << 17;
+        out.extend_from_slice(&state.wrapping_mul(0x2545f4914f6cdd1d).to_le_bytes());
+    }
+    out.truncate(len);
+    out
+}
+
+/// Protocol 1.1 bin-encoding proof workload (LAB-764 / LAB-866): 64 MiB
+/// incompressible payload, where compressed_data dominates the envelope and
+/// the array-of-ints vs msgpack-bin difference is fully visible. Measures the
+/// envelope codec in isolation (rmp encode/decode of StorageEnvelope) and the
+/// full store()/retrieve() e2e paths.
+fn bench_envelope_codec_64mib(c: &mut Criterion) {
+    const SIZE: usize = 64 * 1024 * 1024;
+    let storage = ByteStorage::new(None);
+    let payload = incompressible(SIZE);
+    let wire = storage.store(&payload, None).unwrap();
+    let envelope: StorageEnvelope = rmp_serde::from_slice(&wire).unwrap();
+    // Self-enforcing workload check: LZ4 overhead is positive on incompressible
+    // input, so a generator regression toward compressible data fails here
+    // instead of silently benchmarking the wrong workload.
+    assert!(
+        envelope.compressed_data.len() >= SIZE,
+        "bench payload must be incompressible"
+    );
+    eprintln!(
+        "64mib_incompressible: compressed_data {} B, envelope wire {} B (ratio {:.4})",
+        envelope.compressed_data.len(),
+        wire.len(),
+        wire.len() as f64 / envelope.compressed_data.len() as f64
+    );
+
+    let mut group = c.benchmark_group("byte_storage/64mib_incompressible");
+    group.sample_size(10);
+    group.throughput(Throughput::Bytes(SIZE as u64));
+    group.bench_function("envelope_encode", |b| {
+        b.iter(|| black_box(rmp_serde::to_vec(black_box(&envelope)).unwrap()));
+    });
+    group.bench_function("envelope_decode", |b| {
+        b.iter(|| black_box(rmp_serde::from_slice::<StorageEnvelope>(black_box(&wire)).unwrap()));
+    });
+    group.bench_function("store_e2e", |b| {
+        b.iter(|| black_box(storage.store(black_box(&payload), None).unwrap()));
+    });
+    group.bench_function("retrieve_e2e", |b| {
+        b.iter(|| black_box(storage.retrieve(black_box(&wire)).unwrap()));
+    });
+    group.finish();
+}
+
 fn bench_byte_storage_roundtrip(c: &mut Criterion) {
     let storage = ByteStorage::new(None);
     let mut group = c.benchmark_group("byte_storage/roundtrip");
@@ -54,5 +112,10 @@ fn bench_encrypt_decrypt(c: &mut Criterion) {
     group.finish();
 }
 
-criterion_group!(benches, bench_byte_storage_roundtrip, bench_encrypt_decrypt);
+criterion_group!(
+    benches,
+    bench_byte_storage_roundtrip,
+    bench_encrypt_decrypt,
+    bench_envelope_codec_64mib
+);
 criterion_main!(benches);

src/byte_storage.rs

@@ -54,6 +54,11 @@ const MAX_COMPRESSION_RATIO: u64 = 1000;
 #[derive(Serialize, Deserialize)]
 pub struct StorageEnvelope {
     /// Compressed payload data
+    ///
+    /// Serialized as msgpack `bin` (protocol 1.1, `envelope-bin-encoding.md`).
+    /// Scope is strictly this field: `checksum` stays array-of-ints by
+    /// normative exclusion (crypto-adjacent surface — MUST NOT flip).
+    #[serde(with = "serde_bytes")]
     pub compressed_data: Vec<u8>,
     /// xxHash3-64 checksum for integrity (8 bytes)
     pub checksum: [u8; 8],

tests/dual_decode.rs

@@ -2,45 +2,42 @@
 //!
 //! Protocol 1.1 (`protocol/decisions/envelope-bin-encoding.md`) flips
 //! `compressed_data` from the legacy msgpack array-of-integers encoding to
-//! msgpack `bin` — readers first. This file is the readers-first CI proof:
-//! under the locked decoder stack (rmp-serde + serde_bytes), BOTH reader
-//! shapes accept BOTH encodings, so `bin` envelopes written by 1.1+ writers
-//! are readable today and legacy envelopes stay readable forever.
+//! msgpack `bin` — readers first, then the writer (LAB-866, shipped). This
+//! file is the permanent CI proof: under the locked decoder stack
+//! (rmp-serde with serde_bytes), BOTH reader shapes accept BOTH encodings,
+//! so `bin` envelopes from 1.1+ writers are readable by pre-1.1 readers and
+//! legacy envelopes stay readable forever.
 //!
 //! The full matrix, executed against the byte-pinned protocol vectors:
 //!
-//! | Reader                             | legacy wire (array) | new wire (bin) |
-//! |------------------------------------|---------------------|----------------|
-//! | old (plain `Vec<u8>`, shipped)     | status quo          | proven here    |
-//! |   …incl. `ByteStorage::retrieve()` |                     | proven here    |
-//! | new (`serde_bytes`, writer-flip)   | proven here         | proven here    |
+//! | Reader                              | legacy wire (array) | new wire (bin) |
+//! |-------------------------------------|---------------------|----------------|
+//! | legacy (plain `Vec<u8>`, pre-1.1)   | proven here         | proven here    |
+//! | shipped (`serde_bytes`, 1.1 writer) | proven here         | proven here    |
+//! |   …incl. `ByteStorage::retrieve()`  | proven here         | proven here    |
 //!
 //! Seeded from the LAB-764 toolchain experiment (`dual_decode_experiment.rs`,
-//! attached to the decision memo on that issue); the throughput half of the
-//! experiment is deliberately absent — benchmark evidence belongs to the
-//! writer-flip stage (LAB-510 harness), not this test-only release.
+//! attached to the decision memo on that issue); throughput evidence lives in
+//! the LAB-510 harness (`benches/hot_path.rs`, 64 MiB incompressible group).
 //!
 //! Width boundaries: the fixture's `*_bin` twins all fit in a bin8 header
 //! (`0xc4`, ≤ 255 B). The `width_boundary_*` tests cover the bin16 (`0xc5`)
-//! and bin32 (`0xc6`) headers with real LZ4 output, per the MUST recorded in
-//! the decision record — fixture-level width vectors land with the writer
-//! flip, where the real writer can generate them natively.
+//! and bin32 (`0xc6`) headers with real LZ4 output from the real writer, per
+//! the MUST recorded in the decision record.
 
 #![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
 
 use cachekit_core::{ByteStorage, StorageEnvelope};
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 
 const FIXTURE: &str = include_str!("vectors/wire-format.json");
 
-/// Post-writer-flip twin of StorageEnvelope: identical shape, `serde_bytes`
-/// on `compressed_data` only. `checksum: [u8; 8]` deliberately stays
-/// array-of-ints — normative scope exclusion in the decision record
-/// (crypto-adjacent surface; implementations MUST NOT flip it
-/// opportunistically).
-#[derive(Serialize, Deserialize, PartialEq, Debug)]
-struct StorageEnvelopeV2 {
-    #[serde(with = "serde_bytes")]
+/// Pre-1.1 reader stand-in: identical shape to StorageEnvelope but with a
+/// plain `Vec<u8>` (no `serde_bytes`) on `compressed_data` — the shape every
+/// pre-writer-flip SDK shipped. Kept forever: these matrix legs are the proof
+/// that legacy readers accept `bin` wire and that legacy wire still decodes.
+#[derive(Deserialize, PartialEq, Debug)]
+struct StorageEnvelopeLegacy {
     compressed_data: Vec<u8>,
     checksum: [u8; 8],
     original_size: u32,
@@ -67,17 +64,17 @@ fn load_vectors() -> Vec<Vector> {
 }
 
 /// Assert one envelope's bytes decode identically through every reader shape:
-/// old struct, new struct, and the full shipped `retrieve()` path (checksum
-/// validation + decompression-ratio guard included).
+/// legacy struct, shipped struct, and the full shipped `retrieve()` path
+/// (checksum validation + decompression-ratio guard included).
 fn assert_all_readers_decode(name: &str, wire: &[u8], expected_payload: &[u8]) {
-    let old: StorageEnvelope = rmp_serde::from_slice(wire)
-        .unwrap_or_else(|e| panic!("[{name}] old reader (plain Vec<u8>) failed: {e}"));
-    let new: StorageEnvelopeV2 = rmp_serde::from_slice(wire)
-        .unwrap_or_else(|e| panic!("[{name}] new reader (serde_bytes) failed: {e}"));
-    assert_eq!(old.compressed_data, new.compressed_data, "[{name}]");
-    assert_eq!(old.checksum, new.checksum, "[{name}]");
-    assert_eq!(old.original_size, new.original_size, "[{name}]");
-    assert_eq!(old.format, new.format, "[{name}]");
+    let legacy: StorageEnvelopeLegacy = rmp_serde::from_slice(wire)
+        .unwrap_or_else(|e| panic!("[{name}] legacy reader (plain Vec<u8>) failed: {e}"));
+    let shipped: StorageEnvelope = rmp_serde::from_slice(wire)
+        .unwrap_or_else(|e| panic!("[{name}] shipped reader (serde_bytes) failed: {e}"));
+    assert_eq!(legacy.compressed_data, shipped.compressed_data, "[{name}]");
+    assert_eq!(legacy.checksum, shipped.checksum, "[{name}]");
+    assert_eq!(legacy.original_size, shipped.original_size, "[{name}]");
+    assert_eq!(legacy.format, shipped.format, "[{name}]");
 
     let storage = ByteStorage::new(None);
     let (payload, _format) = storage
@@ -90,9 +87,9 @@ fn assert_all_readers_decode(name: &str, wire: &[u8], expected_payload: &[u8]) {
 }
 
 /// The 4-cell matrix over the legacy pinned vectors: decode each with both
-/// reader shapes, re-encode through the new (serde_bytes) shape to produce
-/// bin wire, and prove that bin wire decodes through both reader shapes AND
-/// today's shipped `retrieve()`.
+/// reader shapes, re-encode through the shipped (serde_bytes) writer shape to
+/// produce bin wire, and prove that bin wire decodes through both reader
+/// shapes AND the shipped `retrieve()`.
 #[test]
 fn dual_decode_matrix_against_legacy_vectors() {
     let vectors = load_vectors();
@@ -112,11 +109,11 @@ fn dual_decode_matrix_against_legacy_vectors() {
         // (the readers-first claim), plus retrieve() on the pinned bytes.
         assert_all_readers_decode(&v.name, &old_wire, &input);
 
-        // Produce new wire the way the flipped writer will.
-        let new_from_old: StorageEnvelopeV2 = rmp_serde::from_slice(&old_wire)
-            .unwrap_or_else(|e| panic!("[{}] new reader must decode legacy wire: {e}", v.name));
+        // Produce new wire the way the shipped writer does.
+        let new_from_old: StorageEnvelope = rmp_serde::from_slice(&old_wire)
+            .unwrap_or_else(|e| panic!("[{}] shipped reader must decode legacy wire: {e}", v.name));
         let new_wire = rmp_serde::to_vec(&new_from_old)
-            .unwrap_or_else(|e| panic!("[{}] serde_bytes shape must serialize: {e}", v.name));
+            .unwrap_or_else(|e| panic!("[{}] shipped shape must serialize: {e}", v.name));
 
         // compressed_data (element [0], right after the 0x94 fixarray marker)
         // must now carry a bin marker.
@@ -141,8 +138,9 @@ fn dual_decode_matrix_against_legacy_vectors() {
         // 3 B array16 header, so bin8's 2 B header shrinks it. Measured worst
         // case on the pinned set is exactly +1 (`empty`, 25 -> 26 B); every
         // other vector ties or shrinks. This +1 B ceiling is the micro-
-        // regression contract the writer-flip step (LAB-764) will read, so keep
-        // it tight — a +2/+3 bloat regression must not slip through green.
+        // regression contract accepted by the writer flip (LAB-764/LAB-866),
+        // so keep it tight — a +2/+3 bloat regression must not slip through
+        // green.
         assert!(
             new_wire.len() <= old_wire.len() + 1,
             "[{}] new wire exceeds the +1 B bin8-header ceiling",
@@ -155,11 +153,10 @@ fn dual_decode_matrix_against_legacy_vectors() {
     }
 }
 
-/// The pinned `*_bin` twins through the same matrix, plus the writer-flip
-/// forecast: re-encoding a twin through the serde_bytes shape must be
-/// byte-identical to the protocol's pinned bin bytes — the writer-flip diff
-/// will point `vectors_reencode_byte_identical` at this set, and this assert
-/// guarantees that switch lands green.
+/// The pinned `*_bin` twins through the same reader matrix. Re-encode
+/// byte-identity for this set lives in `tests/wire_format_vectors.rs`
+/// (`vectors_reencode_byte_identical` at store() level,
+/// `envelope_codec_roundtrip_byte_identical` at codec level).
 #[test]
 fn dual_decode_matrix_against_bin_vectors() {
     let vectors = load_vectors();
@@ -176,17 +173,6 @@ fn dual_decode_matrix_against_bin_vectors() {
             .unwrap_or_else(|e| panic!("[{}] input_hex must decode: {e}", v.name));
 
         assert_all_readers_decode(&v.name, &wire, &input);
-
-        let env: StorageEnvelopeV2 = rmp_serde::from_slice(&wire)
-            .unwrap_or_else(|e| panic!("[{}] new reader must decode bin wire: {e}", v.name));
-        let reencoded = rmp_serde::to_vec(&env)
-            .unwrap_or_else(|e| panic!("[{}] serde_bytes shape must serialize: {e}", v.name));
-        assert_eq!(
-            hex::encode(&reencoded),
-            hex::encode(&wire),
-            "[{}] serde_bytes re-encode is not byte-identical to the pinned bin vector",
-            v.name
-        );
     }
 }
 
@@ -206,22 +192,16 @@ fn incompressible(len: usize) -> Vec<u8> {
 }
 
 /// Build a real envelope (real LZ4 output) from an incompressible payload,
-/// bin-encode it via the serde_bytes shape, and assert the expected bin width
-/// marker before running the full reader matrix on it.
+/// bin-encode it via the shipped writer shape, and assert the expected bin
+/// width marker before running the full reader matrix on it.
 fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) {
     let payload = incompressible(payload_len);
     let env = StorageEnvelope::new(&payload, "msgpack".to_string())
         .unwrap_or_else(|e| panic!("[{name}] envelope construction failed: {e:?}"));
     let compressed_len = env.compressed_data.len();
 
-    let v2 = StorageEnvelopeV2 {
-        compressed_data: env.compressed_data,
-        checksum: env.checksum,
-        original_size: env.original_size,
-        format: env.format,
-    };
-    let wire = rmp_serde::to_vec(&v2)
-        .unwrap_or_else(|e| panic!("[{name}] serde_bytes shape must serialize: {e}"));
+    let wire = rmp_serde::to_vec(&env)
+        .unwrap_or_else(|e| panic!("[{name}] shipped shape must serialize: {e}"));
     assert_eq!(wire[0], 0x94, "[{name}] outer fixarray(4) preserved");
     assert_eq!(
         wire[1], expected_marker,

tests/wire_format_vectors.rs

@@ -12,8 +12,8 @@
 //! their `*_bin` twins (`"envelope_encoding": "bin"`, canonical for 1.1+
 //! writers). Decode-identity runs against BOTH sets, forever. Re-encode
 //! byte-identity runs against the set matching this crate's CURRENT writer
-//! encoding — legacy today; the writer-flip diff (`serde_bytes` on
-//! `compressed_data`) switches those assertions to the `*_bin` set.
+//! encoding — `bin` since the writer flip (`serde_bytes` on
+//! `compressed_data`, LAB-866), so those assertions target the `*_bin` set.
 //!
 //! Fixture provenance: vendored from
 //! <https://github.com/cachekit-io/protocol> `test-vectors/wire-format.json`
@@ -63,8 +63,8 @@ struct Vector {
 }
 
 impl Vector {
-    /// Encoded with the legacy array-of-integers `compressed_data` — the
-    /// encoding this crate's writer currently emits.
+    /// Encoded with the legacy array-of-integers `compressed_data` — what
+    /// pre-1.1 writers emitted; retained forever as legacy-read proof.
     fn is_legacy(&self) -> bool {
         self.envelope_encoding.is_none()
     }
@@ -165,13 +165,17 @@ fn vectors_decode_to_expected_payload() {
 /// and cross-version reads are at risk — regenerate vectors deliberately in
 /// the protocol repo, never adjust expectations here.
 ///
-/// Legacy vectors only: this asserts against the encoding the writer CURRENTLY
-/// emits. The writer-flip diff switches this filter to the `*_bin` set in the
-/// same PR that adds `serde_bytes` to `compressed_data`.
+/// `*_bin` vectors only: this asserts against the encoding the writer
+/// CURRENTLY emits — msgpack `bin` since the writer flip (`serde_bytes` on
+/// `compressed_data`, LAB-866).
 #[test]
 fn vectors_reencode_byte_identical() {
     let storage = ByteStorage::new(None);
-    for vector in load_fixture().vectors.into_iter().filter(Vector::is_legacy) {
+    for vector in load_fixture()
+        .vectors
+        .into_iter()
+        .filter(|v| !v.is_legacy())
+    {
         let input = hex::decode(&vector.input_hex).expect("input_hex must decode");
         let expected = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode");
 
@@ -193,12 +197,16 @@ fn vectors_reencode_byte_identical() {
 /// codec test failing too means the MessagePack layout changed (protocol#11
 /// territory); codec test passing means the LZ4 block encoding changed.
 ///
-/// Legacy vectors only, same reason as `vectors_reencode_byte_identical`:
-/// re-serialization emits the writer's current encoding, so byte-identity can
-/// only hold for the set that matches it.
+/// `*_bin` vectors only, same reason as `vectors_reencode_byte_identical`:
+/// re-serialization emits the writer's current encoding (`bin`), so
+/// byte-identity can only hold for the set that matches it.
 #[test]
 fn envelope_codec_roundtrip_byte_identical() {
-    for vector in load_fixture().vectors.into_iter().filter(Vector::is_legacy) {
+    for vector in load_fixture()
+        .vectors
+        .into_iter()
+        .filter(|v| !v.is_legacy())
+    {
         let canonical = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode");
         let envelope: StorageEnvelope = rmp_serde::from_slice(&canonical)
             .unwrap_or_else(|e| panic!("[{}] envelope must deserialize: {e}", vector.name));