diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index f19aafd..b4f3813 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -29,12 +29,14 @@ jobs: run: | python3 tools/interop-reference.py verify python3 tools/encryption-verify.py + python3 tools/wire-format-reference.py verify - name: Python reference verify (optional deps — AES-GCM seal + msgpack third-encoder conformance) run: | pip install cryptography==49.0.0 msgpack==1.2.1 python3 tools/interop-reference.py verify python3 tools/encryption-verify.py --require-seal + python3 tools/wire-format-reference.py verify - name: JS cross-check (independent encoder + @noble/hashes + WebCrypto) run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index d4c140b..9be360b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ All notable changes to the CacheKit Protocol Specification. ### Specs +- StorageEnvelope `compressed_data` canonical encoding flipped from MessagePack + array-of-ints to `bin` (LAB-783 / + [cachekit-core#54](https://github.com/cachekit-io/cachekit-core/issues/54)): + protocol 1.1+ writers MUST emit `bin`; readers MUST accept both encodings + permanently. **Not a breaking change** — dual-read is mutual in both directions + under rmp-serde, toolchain-verified; no version field or discriminator. + `checksum` stays array-of-ints; `format` untouched. Tiny envelopes may grow + ≤ +1 B; incompressible payloads shrink ~35%. Rationale, evidence, and rollout + order in [decisions/envelope-bin-encoding.md](decisions/envelope-bin-encoding.md). + - Key rotation specified (not yet implemented, LAB-516): client-side keyring — one forward-only current key plus ≤3 decrypt-only master keys; fingerprint-based key selection where per-entry identity exists (cachekit-py frames), sequential @@ -24,6 +34,13 @@ All notable changes to the CacheKit Protocol Specification. ### Test Vectors +- 6 `bin`-encoded twins appended to `test-vectors/wire-format.json` (append-only; + the 6 legacy vectors are retained forever as legacy-read proof; fixture + 1.0.0 → 1.1.0). Twins generated by the stdlib-only + `tools/wire-format-reference.py` and byte-verified against rmp-serde output; + `verify` now runs in CI (stdlib pass + `msgpack` third-encoder conformance) — + the wire-format fixture's first protocol-side CI verification. + - 33 interop key vectors (including every `*16`-tier MessagePack width boundary), 4 value vectors, 1 AAD vector, 1 full HKDF→AES-256-GCM encryption round-trip vector, and 9 must-reject error vectors (`test-vectors/interop-mode.json`). diff --git a/decisions/envelope-bin-encoding.md b/decisions/envelope-bin-encoding.md new file mode 100644 index 0000000..738515d --- /dev/null +++ b/decisions/envelope-bin-encoding.md @@ -0,0 +1,181 @@ +**[Protocol](../README.md)** > **Decisions** > **Envelope `bin` Encoding** + +# Decision Record: StorageEnvelope `compressed_data` → MessagePack `bin` Encoding + +| | | +| :--- | :--- | +| **Status** | Proposed (accepted on merge) | +| **Date** | 2026-07-25 | +| **Ticket** | LAB-783 (RFC) under tracking parent LAB-764 · [cachekit-core#54](https://github.com/cachekit-io/cachekit-core/issues/54) | +| **Normative spec** | [`spec/wire-format.md` → Byte Layout / Encoding compatibility](../spec/wire-format.md#byte-layout-canonical-encoding) — the spec owns the rules; this record owns the rationale, evidence, and rollout order. | +| **Implementation** | Not yet shipped — staged behind this RFC as LAB-764 sub-issues: one-attribute `cachekit-core` diff (`#[serde(with = "serde_bytes")]` on `compressed_data`), fixture re-pin, py wheel + ts NAPI/WASM rebuilds, benchmark proof. | + +--- + +## Context + +`StorageEnvelope.compressed_data` is a plain `Vec` in `cachekit-core`, so +`rmp_serde::to_vec` routes it through Serde's `serialize_seq` and emits a +MessagePack **array of integers** — one element per byte, 2 wire bytes for any +byte ≥ `0x80`. LZ4 output is high-entropy on any input, so roughly half the +bytes pay the 2-byte penalty on **every** enveloped value. Measured +(cachekit-core 0.3.0, 64 MiB incompressible payload, release profile): + +- **Wire size:** 101,189,485 B for a 67,125,251 B LZ4 output — **1.508× + inflation** (the ticket's 1.58× headline at 256 MB, reproduced at this scale). +- **Throughput:** envelope encode ~151–171 MB/s, decode ~123–154 MB/s. The + shipped `store()` end-to-end runs at 128–170 MB/s — per-element MessagePack + encoding dominates, not LZ4 (~GB/s) or xxHash3 (~36 GB/s). +- **Capacity loss:** under cachekit-py's default 100 MB value ceiling, the ~1.5× + inflation turns the effective limit for incompressible values into ~67 MB — + payloads that should fit are rejected. + +Encoding the field as MessagePack `bin` (`0xc4`/`0xc5`/`0xc6`) removes the +inflation entirely (1.0039× — residual is LZ4 incompressible-block overhead) +and speeds the codec up 6.4–7.3× on encode, 8.9–11× on decode. + +## The load-bearing fact: dual-read is mutual, both directions + +The change was originally ticketed as a "breaking wire-format change". It is +not. Under the deployed decoder stack (`rmp-serde` 1.3.1 + `serde_bytes` +0.11.19), **both reader shapes accept both encodings** — see the dual-read +matrix in [`spec/wire-format.md` → Encoding compatibility](../spec/wire-format.md#encoding-compatibility-dual-read). +Mechanically: a pre-flip plain `Vec` reader decodes `bin` wire because +rmp-serde's `bin` branch falls through to `visit_seq` over the buffer when +byte-visiting is not requested, and a post-flip `serde_bytes` reader decodes +legacy wire because `ByteBufVisitor` implements both `visit_seq` and +`visit_bytes`. + +This was first established by source-reading the exact pinned dependency +versions (LAB-764 Feature Design panel), then **executed against the real +toolchain** (LAB-764 decision memo, 2026-07-25): all four cells pass on all six +byte-pinned `wire-format.json` vectors, including the strongest form — `bin` +envelopes fed through today's shipped `ByteStorage::retrieve()` with checksum +validation and decompression-ratio guards intact, returning byte-identical +payloads. The experiment source is attached to the LAB-764 memo and seeds the +implementation PR's unit test. + +Two structural facts make the migration surface small: + +1. **The envelope codec is single-sourced in `cachekit-core`.** cachekit-py + reaches it via FFI; cachekit-ts via NAPI *and* the wasm32 build; no SDK + hand-parses the envelope (code-verified, LAB-764). There is no non-serde + parser anywhere to teach. +2. **cachekit-rs does not use the envelope for values at all** (plain + MessagePack, `rmp_serde::to_vec_named`; core is used only for encryption). + Its value path has zero migration surface. + +## Options considered + +### A. Flip `compressed_data` to `bin`, readers-first — **chosen** + +One serde attribute on one field (`serde_bytes` is already a non-optional +dependency of cachekit-core — the diff adds zero dependencies). No version +field, no discriminator: the MessagePack marker on element `[0]` is +self-describing, and dual-read is proven in both directions. Legacy +array-of-ints remains a permanently accepted read format. + +### B. Versioned envelope (add a version field / discriminator) — rejected + +Strictly worse: the envelope is a bare positional `fixarray(4)` with no version +field to extend, so a discriminator means a 5th element or a new outer shape — +**that** would be the actual breaking change, invalidating every stored +envelope to solve a problem the msgpack type system already solves. Rejected +as inventing an envelope architecture mid-rollout for zero benefit. + +### C. Do nothing / document the ceiling — rejected + +The ~150 MB/s store ceiling and ~1.5× wire tax land on every enveloped write of +poorly-compressible data (blobs, pre-compressed content, high-entropy serialized +values) on a metered-miss-sensitive path, and the capacity loss under the 100 MB +value ceiling rejects payloads that should fit. A 6–11× codec win for a +one-attribute diff clears the bar; the fix only gets more expensive as the +installed base grows. + +## Scope exclusions (explicit, normative in the spec) + +- **`checksum: [u8; 8]` stays array-of-ints.** The win would be 1–7 bytes per + envelope, and it is crypto-adjacent surface (integrity material). Excluded + deliberately — a future RFC may revisit; implementations MUST NOT flip it + opportunistically. +- **`format` is untouched.** It feeds + [AAD v0x03](../spec/encryption.md#additional-authenticated-data-aad) as the + registry token (`"msgpack"`); this change introduces no new token. +- **`original_size` and the outer `fixarray(4)` are unchanged.** + +## Encryption / AAD independence + +Checked against [`spec/encryption.md`](../spec/encryption.md), not assumed: +AAD v0x03 is built **exclusively from metadata** — version byte plus +length-prefixed `tenant_id`, `cache_key`, `format` token, `compressed` token, +and optional `original_type`. **No envelope bytes feed the AAD.** The envelope +bytes are the AES-GCM *plaintext*: flipping the element-`[0]` marker changes a +new entry's ciphertext exactly as any plaintext change would, with no +compatibility consequence — a stored entry decrypts to the envelope encoding it +was written with, and the dual-read rule applies post-decrypt. No re-encryption +of existing entries, no AAD version bump, no new `format` token. + +(This confirms the LAB-764 panel checkpoint. One nuance corrected along the +way: the flip is AAD-independent because AAD contains *no payload-derived +bytes at all* — not because AAD covers "pre-envelope" bytes.) + +## Rollout order (discipline, not necessity) + +Dual-read is already mutual in both directions, so strict ordering is not +load-bearing — but expand/contract costs nothing and removes all reliance on +the fall-through behavior of old readers: + +1. **Readers first:** `cachekit-core` release carrying the dual-decode unit + test (seeded from the LAB-764 experiment) and the re-pinned fixture with the + `*_bin` vectors — proving both encodings decode through `retrieve()` in CI. +2. **Rebuilds:** cachekit-py wheel, cachekit-ts NAPI + wasm32 artifacts pick up + the core release. No SDK code changes — the codec is single-sourced. + cachekit-rs bumps core (encryption-only consumer; value path unaffected). +3. **Writer flip:** the `serde_bytes` attribute lands; `store()` emits `bin`. + Fixture re-encode-identity assertions switch to the `*_bin` vector set in + the same diff (decode-identity keeps running against both sets, forever). + +Benchmark proof (envelope ≈1.0× payload, store-throughput improvement) runs +against the LAB-510 hot-path harness as an implementation acceptance criterion. + +## Documented micro-regression + +The only length tier where `bin` loses is `compressed_data` ≤ 15 bytes, where a +bin8 header (2 B) replaces a fixarray header (1 B) — every longer tier is equal +or smaller, and each payload byte ≥ `0x80` saves a byte. An envelope therefore +grows by **at most 1 byte**, and only when `compressed_data` is ≤ 15 bytes with +no byte ≥ `0x80`; measured `empty` 25 → 26 B, `single_byte` unchanged at 27 B. +Stated here and in the spec so nobody "discovers" it later. + +## Test vectors + +`test-vectors/wire-format.json` is **append-only**: the six legacy vectors are +retained forever as legacy-read proof; six `*_bin` twins (identical fields, +element `[0]` as `bin`) are appended, marked `"envelope_encoding": "bin"` + +`"derived_from"`. Twins are generated by the stdlib-only +[`tools/wire-format-reference.py`](../tools/wire-format-reference.py) — whose +`verify` mode proves codec fidelity by re-encoding the legacy vectors +byte-identically to the rmp-serde-generated pins — and were byte-verified +against real `rmp-serde 1.3.1 + serde_bytes 0.11.19` output. `verify` runs in +this repo's CI (stdlib pass + `msgpack` third-encoder conformance pass), +giving the wire-format fixture its first protocol-side CI verification. +`cachekit-core`'s vendored-fixture sha256 re-pin is a deliberate update +(LAB-423 precedent), not drift; its re-encode byte-identity assertions select +the vector set matching the writer's current encoding (decode-identity keeps +running against both sets, forever). + +**Deferred to the implementation PR:** all six `*_bin` twins exercise the +`bin8` (`0xc4`) header — no legacy vector has `compressed_data` > 255 B, and a +`bin16`/`bin32` twin cannot be derived from the existing pins (a new input +needs real LZ4 output). The implementation PR MUST add at least one +width-boundary vector (> 255 B compressed) generated by the real +`cachekit-core` writer, so third-party readers get fixture coverage of the +larger `bin` headers. + +## Links + +- Normative spec: [`spec/wire-format.md`](../spec/wire-format.md) +- GitHub: [cachekit-io/cachekit-core#54](https://github.com/cachekit-io/cachekit-core/issues/54) +- Multica: LAB-783 (this RFC), LAB-764 (tracking parent; toolchain experiment + + decision memo), LAB-423 (fixture sha-pinning precedent), LAB-510 (benchmark + harness), protocol#11 / PR #18 (envelope facts corrected) diff --git a/spec/wire-format.md b/spec/wire-format.md index d1bfc0b..11aa127 100644 --- a/spec/wire-format.md +++ b/spec/wire-format.md @@ -6,7 +6,7 @@ **LZ4 compression + xxHash3-64 integrity wrapping for cached payloads that use the envelope.** -*Protocol Version 1.0 · Verified against `cachekit-core` v0.3.0 (`src/byte_storage.rs`); envelope test vectors generated at v0.2.0 and unchanged since* +*Protocol Version 1.1 · Verified against `cachekit-core` v0.3.0 (`src/byte_storage.rs`); legacy envelope test vectors generated at v0.2.0 and unchanged since — `bin`-encoded twins added in protocol 1.1 ([decisions/envelope-bin-encoding.md](../decisions/envelope-bin-encoding.md))* @@ -66,26 +66,51 @@ StorageEnvelope { ### Byte Layout (canonical encoding) `rmp_serde::to_vec` encodes the struct **positionally** — a 4-element MessagePack -**array**, not a named map. And because Serde routes `Vec` / `[u8; 8]` through -`serialize_seq`, the two byte fields are encoded as **MessagePack arrays of -integers**, not `bin`: +**array**, not a named map: ```text -┌───────────────────────────────────────────────────────────────────┐ -│ MessagePack Array (4 elements) │ -├───────────────────┬───────────────────────────────────────────────┤ -│ [0] compressed_data │ LZ4 block bytes, one int each │ -│ [1] checksum │ xxHash3-64, big-endian │ -│ [2] original_size │ bytes before compression │ -│ [3] format │ e.g. "msgpack" │ -└───────────────────┴───────────────────────────────────────────────┘ +┌─────────────────────────────────────────────────────────────────────┐ +│ MessagePack Array (4 elements) │ +├───────────────────┬─────────────────────────────────────────────────┤ +│ [0] compressed_data │ LZ4 block bytes (0xc4/0xc5/0xc6) │ +│ [1] checksum │ xxHash3-64, big-endian │ +│ [2] original_size │ bytes before compression │ +│ [3] format │ e.g. "msgpack" │ +└───────────────────┴─────────────────────────────────────────────────┘ ``` -Worked example — the `simple_string` vector from +As of **protocol 1.1**, writers MUST encode `compressed_data` (element `[0]`) as +MessagePack **`bin`** (`0xc4`/`0xc5`/`0xc6`, shortest form). Readers MUST **also** +accept the legacy encoding below — a stored envelope never expires on a schedule, +so legacy-read support is permanent. + +> [!NOTE] +> **Implementation status:** no shipped release emits the canonical `bin` +> encoding yet — `cachekit-core` v0.3.0 (current) still writes the legacy +> encoding. The writer flip is tracked in LAB-764 / +> [cachekit-core#54](https://github.com/cachekit-io/cachekit-core/issues/54). + +`checksum` (element `[1]`) is **deliberately excluded** from the `bin` encoding: it +stays an array of 8 integers. The saving would be 1–7 bytes per envelope, and the +field is crypto-adjacent surface — not worth touching +([decisions/envelope-bin-encoding.md](../decisions/envelope-bin-encoding.md)). +`original_size`, `format`, and the outer `fixarray(4)` are likewise unchanged. + +#### Legacy element[0] encoding (pre-1.1 writers) + +Because the pre-1.1 `StorageEnvelope` routed `Vec` through Serde's +`serialize_seq`, `compressed_data` was encoded as a **MessagePack array of +integers** — one element per byte, 2 bytes on the wire for any byte ≥ `0x80`. +The envelope's logical contents are identical in both encodings; motivation and +measurements live in +[decisions/envelope-bin-encoding.md](../decisions/envelope-bin-encoding.md). + +Worked example — the `simple_string` vector pair from [`test-vectors/wire-format.json`](../test-vectors/wire-format.json), input -`"hello, cachekit!"` (16 bytes): +`"hello, cachekit!"` (16 bytes), in both encodings: ```text +Legacy (array-of-ints, pre-1.1 writers — vector `simple_string`, 44 B): 94 fixarray(4) dc 0012 array16(18) compressed_data cc f0 uint8 240 LZ4 token (15 literals + ext) @@ -96,15 +121,75 @@ Worked example — the `simple_string` vector from 6d cc8a 34 ccb6 3a 3c 52 ccd3 6d 8a 34 b6 3a 3c 52 d3 10 fixint 16 original_size a7 6d 73 67 70 61 63 6b fixstr(7) format = "msgpack" + +Canonical (bin, 1.1+ writers — vector `simple_string_bin`, 42 B): +94 fixarray(4) + c4 12 bin8(18) compressed_data + f0 01 68 65 6c 6c 6f 2c raw bytes same 18 LZ4 bytes, + 20 63 61 63 68 65 6b 69 74 21 1 byte each on the wire + 98 fixarray(8) checksum (unchanged) + 6d cc8a 34 ccb6 3a 3c 52 ccd3 6d 8a 34 b6 3a 3c 52 d3 + 10 fixint 16 original_size (unchanged) + a7 6d 73 67 70 61 63 6b fixstr(7) format = "msgpack" (unchanged) ``` +### Encoding compatibility (dual-read) + +The two encodings are **mutually intelligible in both directions** under +`rmp-serde` — this is a property of the deployed readers, not a migration +promise. Toolchain-verified (rmp-serde 1.3.1, serde_bytes 0.11.19, rmp 0.8.15, +serde 1.0.228) on all six byte-pinned vectors, including `bin` wire fed through +the shipped `ByteStorage::retrieve()` with checksum validation and +decompression-ratio guards intact (LAB-764): + +| Reader | Legacy wire (array-of-ints) | Canonical wire (`bin`) | +| :--- | :---: | :---: | +| Pre-flip (plain `Vec`, shipped today) | ✅ status quo | ✅ | +| Post-flip (`serde_bytes`) | ✅ | ✅ | + +Consequences: + +- **This is not a breaking change, and no version field or discriminator is + introduced.** The MessagePack marker on element `[0]` is self-describing; + the outer envelope shape is unchanged. +- The envelope codec is **single-sourced in `cachekit-core`** — every SDK + (py via FFI, ts via NAPI and wasm32) reaches it through the Rust core, and + `cachekit-rs` does not use the envelope for values at all. No SDK hand-parses + the envelope. Any future non-`rmp-serde` implementation is bound by the same + reader requirement stated under [Byte Layout](#byte-layout-canonical-encoding). +- **Encrypted entries are unaffected structurally**: the envelope bytes are the + AES-GCM *plaintext*, and [AAD v0x03](encryption.md#additional-authenticated-data-aad) + is built exclusively from metadata (`tenant_id`, `cache_key`, `format` token, + `compressed` token, optional `original_type`) — no envelope bytes feed the AAD, + and the flip introduces no new `format` token. A stored entry decrypts to the + envelope encoding it was written with; the dual-read rule then applies. No + re-encryption, no AAD change. + +> [!NOTE] +> **Size micro-regression on tiny envelopes.** The only length tier where `bin` +> loses is `compressed_data` ≤ 15 bytes (bin8 header 2 B vs fixarray 1 B), so an +> envelope can **grow by at most 1 byte** — and only when no payload byte is +> ≥ `0x80` (measured: the `empty` vector grows 25 → 26 B; `single_byte` is +> unchanged at 27 B). Every other envelope shrinks or stays equal; measured wins +> are in [decisions/envelope-bin-encoding.md](../decisions/envelope-bin-encoding.md). + +Both encodings are pinned by +[`test-vectors/wire-format.json`](../test-vectors/wire-format.json): the legacy +vectors are retained **forever** as legacy-read proof, and each has a `*_bin` +twin appended in protocol 1.1 (marked `"envelope_encoding": "bin"`). The fixture +is append-only and verified in this repo's CI by +[`tools/wire-format-reference.py`](../tools/wire-format-reference.py); vector +provenance and the downstream re-pin plan live in +[decisions/envelope-bin-encoding.md](../decisions/envelope-bin-encoding.md). + > [!WARNING] -> **Earlier revisions of this document described the envelope as a MessagePack map -> with `bin`-encoded byte fields. That was never what `cachekit-core` emitted** — the -> published test vectors (generated from the implementation) have always used the -> positional-array encoding above, and the TypeScript SDK's protocol tests verify it -> byte-for-byte against Python. The array encoding is authoritative and canonical. -> Writers MUST emit it. +> **History.** Earlier revisions of this document described the envelope as a +> MessagePack *map* with `bin`-encoded byte fields — **that was never what +> `cachekit-core` emitted** (the envelope has always been the positional array +> above; protocol#11 corrected the prose). Writers emitted the array-of-ints +> element-`[0]` encoding through protocol 1.0; protocol 1.1 makes `bin` the +> canonical writer encoding for `compressed_data` only. `checksum` remains an +> array of integers in **both** revisions — a reader MUST NOT expect `bin` there. > [!WARNING] > **Discrepancy with RFC** — The RFC (Section 4.3.3) states the checksum is **Blake3 (32 bytes)**. The actual `cachekit-core` implementation uses **xxHash3-64 (8 bytes)**. The crate comments explain: "xxHash3-64 checksums for corruption detection (19x faster than Blake3)". xxHash3-64 is non-cryptographic — tamper resistance is provided by the encryption layer (AES-GCM auth tag), not the checksum. **The implementation (xxHash3-64) is authoritative.** @@ -176,6 +261,11 @@ let checksum: [u8; 8] = xxh3_64(&original_data).to_be_bytes(); > [!IMPORTANT] > All three limits below MUST be enforced by every implementation of the ByteStorage envelope. The decompression bomb check uses integer arithmetic — do not substitute floating-point. +> Additionally, a decoder MUST validate any declared MessagePack `bin`/array +> length header against the remaining input bytes **before** allocating for it — +> a 5-byte `bin32` header can otherwise declare a 4 GiB allocation from a +> ~30-byte envelope. (Slice-based decoders such as `rmp-serde` satisfy this +> inherently; readers that pre-allocate from length fields must check.) | Limit | Value | Purpose | | :--- | ---: | :--- | @@ -219,7 +309,7 @@ Input: raw_data (bytes), format (string, default "msgpack") original_size: raw_data.length, format: format } -6. Serialize: envelope_bytes = msgpack_encode(envelope) +6. Serialize: envelope_bytes = msgpack_encode(envelope) // compressed_data as bin (1.1+) 7. Validate: envelope_bytes.length <= 512 MB 8. Return: envelope_bytes ``` @@ -238,6 +328,7 @@ Input: envelope_bytes 1. Validate: envelope_bytes.length <= 512 MB 2. Deserialize: envelope = msgpack_decode(envelope_bytes) as StorageEnvelope + // accept BOTH element[0] encodings: bin AND array-of-ints 3. Validate: envelope.compressed_data.length <= 512 MB 4. Validate: envelope.original_size <= 512 MB 5. Bomb check: (see Security Limits above) diff --git a/test-vectors/wire-format.json b/test-vectors/wire-format.json index dde964c..9be2ba0 100644 --- a/test-vectors/wire-format.json +++ b/test-vectors/wire-format.json @@ -1,8 +1,8 @@ { "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input", "compression": "LZ4 block format (NOT frame)", - "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]; byte fields encode as msgpack arrays of integers, not bin", - "generator": "cachekit-core v0.2.0", + "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]. Vectors without an 'envelope_encoding' field use the legacy array-of-integers encoding for compressed_data and are retained forever as legacy-read proof; their '*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). checksum always encodes as an array of 8 integers. Normative rules: spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'.", + "generator": "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)", "limits": { "max_compressed_size": 536870912, "max_compression_ratio": 1000, @@ -62,7 +62,73 @@ "input_hex": "ff", "input_size": 1, "name": "single_byte" + }, + { + "description": "bin-encoded twin of 'empty': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40100982d06cc800538ccd3cc94ccc200a76d73677061636b", + "envelope_size": 26, + "format": "msgpack", + "input_hex": "", + "input_size": 0, + "name": "empty_bin", + "envelope_encoding": "bin", + "derived_from": "empty" + }, + { + "description": "bin-encoded twin of 'simple_string': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c412f00168656c6c6f2c2063616368656b697421986dcc8a34ccb63a3c52ccd310a76d73677061636b", + "envelope_size": 42, + "format": "msgpack", + "input_hex": "68656c6c6f2c2063616368656b697421", + "input_size": 16, + "name": "simple_string_bin", + "envelope_encoding": "bin", + "derived_from": "simple_string" + }, + { + "description": "bin-encoded twin of 'binary_data': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c442f031243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917982d016e0411ccedcc9e3440a76d73677061636b", + "envelope_size": 89, + "format": "msgpack", + "input_hex": "243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917", + "input_size": 64, + "name": "binary_data_bin", + "envelope_encoding": "bin", + "derived_from": "binary_data" + }, + { + "description": "bin-encoded twin of 'large_compressible': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40f1f410100ffffffe960414141414141984d35ccad08cce9ccd77c0dcd0400a76d73677061636b", + "envelope_size": 41, + "format": "msgpack", + "input_hex": "41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141", + "input_size": 1024, + "name": "large_compressible_bin", + "envelope_encoding": "bin", + "derived_from": "large_compressible" + }, + { + "description": "bin-encoded twin of 'json_like': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c44cf03b7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d98cc87673661cc9c4bcca8204aa76d73677061636b", + "envelope_size": 100, + "format": "msgpack", + "input_hex": "7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d", + "input_size": 74, + "name": "json_like_bin", + "envelope_encoding": "bin", + "derived_from": "json_like" + }, + { + "description": "bin-encoded twin of 'single_byte': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40210ff98ccd6ccbcccec3c6b29ccd72e01a76d73677061636b", + "envelope_size": 27, + "format": "msgpack", + "input_hex": "ff", + "input_size": 1, + "name": "single_byte_bin", + "envelope_encoding": "bin", + "derived_from": "single_byte" } ], - "version": "1.0.0" + "version": "1.1.0" } diff --git a/tools/wire-format-reference.py b/tools/wire-format-reference.py new file mode 100644 index 0000000..99da3a5 --- /dev/null +++ b/tools/wire-format-reference.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Reference encoder/verifier for the ByteStorage envelope (spec/wire-format.md). + +Stdlib-only (one optional extra, see below). Scope: the **MessagePack encoding** +of the StorageEnvelope positional array — both the legacy element[0] encoding +(array of integers, pre-1.1 writers) and the canonical one (msgpack `bin`, +protocol 1.1+ writers). LZ4 decompression and xxHash3-64 recomputation are NOT +verified here (neither is stdlib); that enforcement lives in cachekit-core's CI +(`tests/wire_format_vectors.rs`, LAB-423). + +What `verify` proves, for every vector pair in ../test-vectors/wire-format.json: + 1. Codec fidelity — decoding a legacy vector and re-encoding it in legacy form + reproduces the pinned bytes exactly. The legacy bytes were generated by + `rmp_serde::to_vec` (cachekit-core v0.2.0), so byte-identity here proves + this codec makes the same shortest-form header choices as rmp — which is + what makes the derived `bin` encoding trustworthy. + 2. Twin equivalence — each `*_bin` vector decodes to field-identical contents + as its legacy base, its element[0] carries a bin marker (0xc4/0xc5/0xc6), + the outer fixarray(4) is preserved, and re-encoding the base's fields in + bin form reproduces the `*_bin` bytes exactly. + 3. Documented size bound — a bin envelope is never more than 1 byte larger + than its legacy twin (the header-arithmetic bound stated in the spec). + +Usage: + python3 tools/wire-format-reference.py verify # default + python3 tools/wire-format-reference.py generate # (re)derive *_bin vectors + +One optional-dependency check deepens `verify` when importable (runs in CI): + - `msgpack`: third-encoder conformance — msgpack-python re-encodes both forms + from decoded fields and must reproduce the pinned bytes byte-identically. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +FIXTURE_PATH = Path(__file__).resolve().parent.parent / "test-vectors" / "wire-format.json" + +FIXTURE_VERSION = "1.1.0" +ENVELOPE_FORMAT = ( + "MessagePack positional array (rmp_serde::to_vec): " + "[compressed_data, checksum, original_size, format]. Vectors without an " + "'envelope_encoding' field use the legacy array-of-integers encoding for " + "compressed_data and are retained forever as legacy-read proof; their " + "'*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). " + "checksum always encodes as an array of 8 integers. Normative rules: " + "spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'." +) +GENERATOR = ( + "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: " + "tools/wire-format-reference.py generate - deterministic re-encode of the " + "legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 " + "output (LAB-783)" +) + +# --- minimal MessagePack codec ------------------------------------------------ +# Deliberately scoped to the types a StorageEnvelope uses. The encoder emits +# shortest-form headers only — the same choices rmp_serde makes — and `verify` +# proves that equivalence against the pinned legacy bytes on every run. + + +class _Reader: + def __init__(self, buf: bytes): + self.buf = buf + self.pos = 0 + + def take(self, n: int) -> bytes: + if self.pos + n > len(self.buf): + raise ValueError("truncated msgpack document") + out = self.buf[self.pos : self.pos + n] + self.pos += n + return out + + def u8(self) -> int: + return self.take(1)[0] + + def be(self, n: int) -> int: + return int.from_bytes(self.take(n), "big") + + def peek(self) -> int: + """Look at the next marker byte without consuming it (bounds-checked).""" + if self.pos >= len(self.buf): + raise ValueError("truncated msgpack document") + return self.buf[self.pos] + + +def _decode_uint(r: _Reader) -> int: + m = r.u8() + if m <= 0x7F: + return m + if m == 0xCC: + return r.be(1) + if m == 0xCD: + return r.be(2) + if m == 0xCE: + return r.be(4) + raise ValueError(f"expected msgpack uint, got marker 0x{m:02x}") + + +def _decode_array_header(r: _Reader) -> int: + m = r.u8() + if 0x90 <= m <= 0x9F: + return m & 0x0F + if m == 0xDC: + return r.be(2) + if m == 0xDD: + return r.be(4) + raise ValueError(f"expected msgpack array, got marker 0x{m:02x}") + + +def _decode_str(r: _Reader) -> str: + m = r.u8() + if not 0xA0 <= m <= 0xBF: + # format is a bounded registry token (e.g. "msgpack"), always fixstr. + raise ValueError(f"expected msgpack fixstr, got marker 0x{m:02x}") + return r.take(m & 0x1F).decode("utf-8") + + +def _decode_bytes_field(r: _Reader) -> tuple[bytes, str]: + """Decode element[0]: either legacy array-of-ints or bin. Returns (data, encoding).""" + m = r.peek() + if m in (0xC4, 0xC5, 0xC6): + r.u8() + n = r.be({0xC4: 1, 0xC5: 2, 0xC6: 4}[m]) + return r.take(n), "bin" + n = _decode_array_header(r) + data = bytearray() + for _ in range(n): + v = _decode_uint(r) + if v > 0xFF: + raise ValueError(f"byte-array element out of range: {v}") + data.append(v) + return bytes(data), "int-array" + + +def _encode_uint(v: int) -> bytes: + if v < 0: + raise ValueError("negative uint") + if v <= 0x7F: + return bytes([v]) + if v <= 0xFF: + return bytes([0xCC, v]) + if v <= 0xFFFF: + return b"\xcd" + v.to_bytes(2, "big") + if v <= 0xFFFFFFFF: + return b"\xce" + v.to_bytes(4, "big") + raise ValueError("uint too large for envelope fields") + + +def _encode_array_header(n: int) -> bytes: + if n <= 0x0F: + return bytes([0x90 | n]) + if n <= 0xFFFF: + return b"\xdc" + n.to_bytes(2, "big") + return b"\xdd" + n.to_bytes(4, "big") + + +def _encode_int_array(data: bytes) -> bytes: + return _encode_array_header(len(data)) + b"".join(_encode_uint(b) for b in data) + + +def _encode_bin(data: bytes) -> bytes: + n = len(data) + if n <= 0xFF: + return b"\xc4" + bytes([n]) + data + if n <= 0xFFFF: + return b"\xc5" + n.to_bytes(2, "big") + data + return b"\xc6" + n.to_bytes(4, "big") + data + + +def _encode_str(s: str) -> bytes: + raw = s.encode("utf-8") + if len(raw) > 31: + raise ValueError("format string too long for a fixstr registry token") + return bytes([0xA0 | len(raw)]) + raw + + +# --- envelope <-> fields ------------------------------------------------------- + + +def decode_envelope(env: bytes) -> tuple[bytes, bytes, int, str, str]: + """Returns (compressed_data, checksum, original_size, format, element0_encoding).""" + r = _Reader(env) + if _decode_array_header(r) != 4: + raise ValueError("StorageEnvelope must be a 4-element array") + data, encoding = _decode_bytes_field(r) + if _decode_array_header(r) != 8: + raise ValueError("checksum must be an 8-element array") + checksum = bytes(_decode_uint(r) for _ in range(8)) + original_size = _decode_uint(r) + fmt = _decode_str(r) + if r.pos != len(env): + raise ValueError(f"{len(env) - r.pos} trailing bytes after envelope") + return data, checksum, original_size, fmt, encoding + + +def encode_envelope( + data: bytes, checksum: bytes, original_size: int, fmt: str, *, encoding: str +) -> bytes: + element0 = _encode_bin(data) if encoding == "bin" else _encode_int_array(data) + return ( + _encode_array_header(4) + + element0 + + _encode_int_array(checksum) + + _encode_uint(original_size) + + _encode_str(fmt) + ) + + +# --- generate / verify ---------------------------------------------------------- + + +def _load() -> dict: + return json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + + +def _split_vectors(fixture: dict) -> tuple[list[dict], dict[str, dict]]: + legacy = [v for v in fixture["vectors"] if "envelope_encoding" not in v] + bins = {v["name"]: v for v in fixture["vectors"] if v.get("envelope_encoding") == "bin"} + # Fail closed: every vector must classify as exactly one of the two sets, + # with no duplicate names — otherwise verify would silently skip it (and + # generate would silently drop it from the append-only fixture). + if len(legacy) + len(bins) != len(fixture["vectors"]) or len({v["name"] for v in legacy}) != len(legacy): + raise ValueError("fixture contains unclassifiable or duplicate-named vectors") + return legacy, bins + + +def _bin_twin(base: dict) -> dict: + data, checksum, size, fmt, encoding = decode_envelope(bytes.fromhex(base["envelope_hex"])) + assert encoding == "int-array", f"[{base['name']}] base vector is not legacy-encoded" + env = encode_envelope(data, checksum, size, fmt, encoding="bin") + twin = dict(base) # preserves key order of the legacy vectors + twin["name"] = base["name"] + "_bin" + twin["description"] = ( + f"bin-encoded twin of '{base['name']}': identical fields, compressed_data " + "as msgpack bin (canonical writer encoding, protocol 1.1+)" + ) + twin["envelope_hex"] = env.hex() + twin["envelope_size"] = len(env) + twin["envelope_encoding"] = "bin" + twin["derived_from"] = base["name"] + return twin + + +def generate() -> int: + fixture = _load() + legacy, _ = _split_vectors(fixture) + fixture["vectors"] = legacy + [_bin_twin(v) for v in legacy] + fixture["envelope_format"] = ENVELOPE_FORMAT + fixture["generator"] = GENERATOR + fixture["version"] = FIXTURE_VERSION + FIXTURE_PATH.write_text(json.dumps(fixture, indent=2) + "\n", encoding="utf-8") + print(f"wrote {FIXTURE_PATH.name}: {len(legacy)} legacy + {len(legacy)} bin vectors") + return 0 + + +def _verify_vector(base: dict, bins: dict, msgpack) -> str: + """Validate one legacy vector against its bin twin (popped from `bins`). + + Returns the one-line size-delta summary on success, or raises + AssertionError/ValueError (and, on a malformed fixture entry, + IndexError/KeyError) which verify()'s per-vector guard isolates. + """ + old_env = bytes.fromhex(base["envelope_hex"]) + data, checksum, size, fmt, encoding = decode_envelope(old_env) + + # 1. codec fidelity against the rmp_serde-generated legacy bytes + assert encoding == "int-array", "legacy vector not array-of-ints encoded" + assert encode_envelope(data, checksum, size, fmt, encoding="int-array") == old_env, ( + "legacy re-encode not byte-identical" + ) + assert len(old_env) == base["envelope_size"], "envelope_size mismatch" + assert fmt == base["format"], "format field mismatch" + assert size == base["input_size"], "original_size != input_size" + + # 2. twin equivalence + twin = bins.pop(base["name"] + "_bin", None) + assert twin is not None, "missing bin twin vector" + new_env = bytes.fromhex(twin["envelope_hex"]) + assert encode_envelope(data, checksum, size, fmt, encoding="bin") == new_env, ( + "bin re-encode not byte-identical to twin" + ) + assert new_env[0] == 0x94, "outer fixarray(4) not preserved" + assert new_env[1] in (0xC4, 0xC5, 0xC6), "element[0] not bin-encoded" + t_data, t_checksum, t_size, t_fmt, t_encoding = decode_envelope(new_env) + assert t_encoding == "bin" + assert (t_data, t_checksum, t_size, t_fmt) == (data, checksum, size, fmt), ( + "twin decodes to different fields" + ) + assert twin["input_hex"] == base["input_hex"], "twin input_hex drifted" + assert twin["input_size"] == base["input_size"], "twin input_size drifted" + assert twin["format"] == base["format"], "twin format drifted" + assert len(new_env) == twin["envelope_size"], "twin envelope_size mismatch" + + # 3. documented size bound: the only pairing where bin loses is + # bin8 (2 B) vs fixarray (1 B), i.e. compressed_data <= 15 bytes, + # so a bin twin is never more than 1 byte larger than its legacy base. + assert len(new_env) <= len(old_env) + 1, "bin twin exceeds +1 B header bound" + + # optional: third-encoder conformance via msgpack-python + if msgpack is not None: + obj = msgpack.unpackb(new_env, raw=False) + assert obj == [data, list(checksum), size, fmt], "msgpack-python bin decode mismatch" + assert msgpack.packb([data, list(checksum), size, fmt], use_bin_type=True) == new_env, ( + "msgpack-python bin re-encode mismatch" + ) + obj = msgpack.unpackb(old_env, raw=False) + assert obj == [list(data), list(checksum), size, fmt], ( + "msgpack-python legacy decode mismatch" + ) + assert msgpack.packb([list(data), list(checksum), size, fmt]) == old_env, ( + "msgpack-python legacy re-encode mismatch" + ) + + delta = len(new_env) - len(old_env) + return f"legacy {len(old_env)} B -> bin {len(new_env)} B ({delta:+d} B)" + + +def verify() -> int: + fixture = _load() + legacy, bins = _split_vectors(fixture) + if not legacy: + print("FAIL: no legacy vectors found", file=sys.stderr) + return 1 + if fixture.get("version") != FIXTURE_VERSION: + print(f"FAIL: fixture version {fixture.get('version')} != {FIXTURE_VERSION}", file=sys.stderr) + return 1 + + try: + import msgpack # type: ignore[import-untyped] + except ImportError: + msgpack = None + + failures = 0 + for base in legacy: + name = base["name"] + try: + print(f" ok {name}: {_verify_vector(base, bins, msgpack)}") + except (AssertionError, ValueError, IndexError, KeyError) as e: + # Per-vector isolation: a malformed vector (truncated hex, missing + # field) fails only itself with a named FAIL line, not the whole run. + failures += 1 + print(f" FAIL {name}: {e!r}", file=sys.stderr) + + for orphan in bins: + failures += 1 + print(f" FAIL {orphan}: bin vector without a legacy base", file=sys.stderr) + + conformance = "with msgpack-python conformance" if msgpack else "stdlib-only" + if failures: + print(f"FAIL: {failures} failure(s) ({conformance})", file=sys.stderr) + return 1 + print(f"all {len(legacy)} vector pairs verified ({conformance})") + return 0 + + +def main() -> int: + cmd = sys.argv[1] if len(sys.argv) > 1 else "verify" + if cmd == "generate": + return generate() + if cmd == "verify": + return verify() + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main())