From 91288fe133e80bef346e527185a01c317d98f4b8 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 29 Jul 2026 10:15:27 +1000 Subject: [PATCH 1/2] feat(envelope): pick up cachekit-core 0.4.0 (bin-encoded envelopes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump cachekit-core 0.3.0 -> 0.4.0 (LAB-900, protocol 1.1 rollout stage 5). Fresh writes now emit StorageEnvelope compressed_data as MessagePack bin (0xc4/0xc5/0xc6) instead of array-of-ints; legacy envelopes stay readable forever. No py-side code change — the envelope codec is single-sourced in cachekit-core. Byte-verification (tests/unit/protocol/test_envelope_wire_vectors.py, fixture vendored sha256-pinned from cachekit-io/protocol wire-format.json): - legacy-read: all 6 pre-0.4.0 array-of-ints vectors decode via the real FFI retrieve path, and a transcoded legacy envelope is served through the full decorator retrieve path (FileBackend L2, no recompute) - bin-emit: fresh decorator writes carry a bin-encoded envelope inside the CK v3 frame; bin8/bin16/bin32 width tiers pinned against the real writer (the protocol *_bin pins only cover bin8) - re-encode identity: the 0.4.0 writer reproduces all 6 *_bin protocol pins byte-identically - round-trip identity through the full stack, compressible + incompressible Docs: DEVELOPMENT.md envelope row replaced — the documented custom [version][format][checksum][...] layout never existed (cachekit-core#46 envelope-doc residual); now describes the real rmp_serde positional fixarray(4) with the 0.4.0 bin encoding and permanent legacy-read rule. Secrets baseline: wire-format.json hex vectors are high-entropy test pins, audited into .secrets.baseline (same convention as the AAD/encryption tests). --- .secrets.baseline | 478 +++++++++++++++++- Cargo.lock | 4 +- DEVELOPMENT.md | 15 +- rust/Cargo.toml | 2 +- tests/unit/protocol/fixtures/wire-format.json | 134 +++++ .../protocol/test_envelope_wire_vectors.py | 234 +++++++++ 6 files changed, 855 insertions(+), 12 deletions(-) create mode 100644 tests/unit/protocol/fixtures/wire-format.json create mode 100644 tests/unit/protocol/test_envelope_wire_vectors.py diff --git a/.secrets.baseline b/.secrets.baseline index 595fd63..7580ed5 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -142,14 +142,23 @@ "filename": "docs/configuration.md", "hashed_secret": "e45470554d7d1790539cc3091db4699ea780fb8f", "is_verified": false, - "line_number": 149 + "line_number": 153 }, { "type": "Hex High Entropy String", "filename": "docs/configuration.md", "hashed_secret": "d88aec4de8c76ada9c0172733e4300bb82abcfc7", "is_verified": false, - "line_number": 480 + "line_number": 543 + } + ], + "docs/features/interop-mode.md": [ + { + "type": "Hex High Entropy String", + "filename": "docs/features/interop-mode.md", + "hashed_secret": "be5aef15f4e2838807bbd3eae0273d810d3b06d9", + "is_verified": false, + "line_number": 138 } ], "docs/features/ssrf-protection.md": [ @@ -174,21 +183,21 @@ "filename": "docs/features/zero-knowledge-encryption.md", "hashed_secret": "d8eab3976a5dca3e6c91149eff8311b31399ecc7", "is_verified": false, - "line_number": 196 + "line_number": 228 }, { "type": "Secret Keyword", "filename": "docs/features/zero-knowledge-encryption.md", "hashed_secret": "b1aa66c32f3e9119bb6d52d55f9e94b1cb8d6cbe", "is_verified": false, - "line_number": 197 + "line_number": 229 }, { "type": "Secret Keyword", "filename": "docs/features/zero-knowledge-encryption.md", "hashed_secret": "4084cee9e75572b8e2e055e114ea7458e3978b5b", "is_verified": false, - "line_number": 422 + "line_number": 572 } ], "docs/serializers/encryption.md": [ @@ -197,14 +206,14 @@ "filename": "docs/serializers/encryption.md", "hashed_secret": "4084cee9e75572b8e2e055e114ea7458e3978b5b", "is_verified": false, - "line_number": 31 + "line_number": 32 }, { "type": "Secret Keyword", "filename": "docs/serializers/encryption.md", "hashed_secret": "9d1a8d8480340f9fbc3e556ab08a6a54b68ca09d", "is_verified": false, - "line_number": 32 + "line_number": 33 } ], "src/cachekit/cache_handler.py": [ @@ -225,6 +234,15 @@ "line_number": 572 } ], + "src/cachekit/serializers/interop_serializer.py": [ + { + "type": "Hex High Entropy String", + "filename": "src/cachekit/serializers/interop_serializer.py", + "hashed_secret": "be5aef15f4e2838807bbd3eae0273d810d3b06d9", + "is_verified": false, + "line_number": 31 + } + ], "tests/critical/test_aad_v03_security.py": [ { "type": "Secret Keyword", @@ -399,6 +417,450 @@ "line_number": 23 } ], + "tests/unit/protocol/fixtures/interop-mode.json": [ + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "8677d2e92b4c4c26aa32448a644b3ddb8f48b2f2", + "is_verified": false, + "line_number": 29 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "d994176f0253fe78052c0b28d5c44a2c10db10b2", + "is_verified": false, + "line_number": 41 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "f6e54dbf4e7fb9833132bcc0d2c7af4173dd1330", + "is_verified": false, + "line_number": 55 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "a8805a8b330e42304195ca4c5c071704b0d0e4f4", + "is_verified": false, + "line_number": 69 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "fcfd2387fae5180f0940a989f790e931e1695e34", + "is_verified": false, + "line_number": 88 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "c350b6b670bf420c6a1c95f0b972eeb28d989377", + "is_verified": false, + "line_number": 107 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "b56b4a08fa225913c8447f29514597561b5b6aff", + "is_verified": false, + "line_number": 118 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "656f686349704b86b3c6b020eb586924094bed75", + "is_verified": false, + "line_number": 119 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "4dbff759506ff7b7be357fbabc37bea9ccdf8b9b", + "is_verified": false, + "line_number": 134 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "0fcf2210617523e17b5190ec2d7bc2d573b59979", + "is_verified": false, + "line_number": 151 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "d4f2eed65576ac519606f79688244a9dcc49001e", + "is_verified": false, + "line_number": 197 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "04d10783bc3242b183c06a79be25e58fa4405d8b", + "is_verified": false, + "line_number": 242 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "47c77092e7c48fad485c9a40c130aed172ae070e", + "is_verified": false, + "line_number": 243 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "44b2949989534c8116af10a43b5502856dcd5ba9", + "is_verified": false, + "line_number": 270 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "e515827ef13f2fb4e25cf5605a0aeffa1ecc0549", + "is_verified": false, + "line_number": 284 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "2666907faa2eee5e43ae71eee94aa6866e7d0cc3", + "is_verified": false, + "line_number": 297 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "76b913f21a6b20b41b55fd683f3c33f3d48eb2d8", + "is_verified": false, + "line_number": 298 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "d46e4e6835383d874f1e18206ada17b35743dae4", + "is_verified": false, + "line_number": 312 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "6c4f110f4fded9d5b156a4b77f68b33dd81f3190", + "is_verified": false, + "line_number": 338 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "3d869c5fe217aa53e9f494b136b60fa843d06144", + "is_verified": false, + "line_number": 352 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "e3441e4a5a89f4844e4f9909068519e76b13a353", + "is_verified": false, + "line_number": 368 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "1c04de001bbd77f1a682f2e7a90875447526e0e1", + "is_verified": false, + "line_number": 382 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "75cb24552348b522b6579d0425043740554ecb8b", + "is_verified": false, + "line_number": 383 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "8ede44f3c12dfd63cf334dd5534cd77c0e34be73", + "is_verified": false, + "line_number": 404 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "9f7e96bf97958de94601f0f3765c98191c02f774", + "is_verified": false, + "line_number": 422 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "c134459fe6b30199a5ab2c219444ed6b46726a68", + "is_verified": false, + "line_number": 440 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "7b6cada8279157007adf8a9047a47ac185c257a3", + "is_verified": false, + "line_number": 460 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "04ab5d27d6fe4a188096409be1299eb1f14f5484", + "is_verified": false, + "line_number": 493 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "c80c3022c1cb3f47fc3139dee12c332d5a411d79", + "is_verified": false, + "line_number": 506 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "08f8c25bc2943572a0453ad25e7b29701f6705f6", + "is_verified": false, + "line_number": 507 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "ac2842d5bb48d0b4aed2ad2a03bd34b70018121e", + "is_verified": false, + "line_number": 520 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "155f47c5b78a4e90113b59abe11867cd8482bea8", + "is_verified": false, + "line_number": 521 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "f95064ae0a021525a4ac09281f84f124b9018023", + "is_verified": false, + "line_number": 548 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "0bcb78e33bb1f929af57e1126b55e708bc573438", + "is_verified": false, + "line_number": 549 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "07982b98f02163bb51f79b513dca40325bf3e43b", + "is_verified": false, + "line_number": 563 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "f70af5f834e12e1917e40a52a6b47ee66db824f2", + "is_verified": false, + "line_number": 576 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "6fb80b90a36c5a47a9a2d067db99da4fe117fb3c", + "is_verified": false, + "line_number": 577 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "5e2103d1c489cc4a8f82e4a10b22190a90df623c", + "is_verified": false, + "line_number": 593 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "45065ee3256a46f7d208fe60ab043388a7942efb", + "is_verified": false, + "line_number": 594 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "be5aef15f4e2838807bbd3eae0273d810d3b06d9", + "is_verified": false, + "line_number": 606 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "e10287fe94b240c3f1b80108a6f38ba327656d1a", + "is_verified": false, + "line_number": 637 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "22451150856583e1d666836db8b8f8069e5180bb", + "is_verified": false, + "line_number": 725 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "43f8a5e00c7a0964cdf90022b9eb2cab0b0f96ca", + "is_verified": false, + "line_number": 734 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/interop-mode.json", + "hashed_secret": "f0890edf4761a1bc77aa54ed51547f2f8bc2bcb3", + "is_verified": false, + "line_number": 741 + } + ], + "tests/unit/protocol/fixtures/wire-format.json": [ + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "519ead85e7442a8a58869964bc9478519fea1950", + "is_verified": false, + "line_number": 14 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "6aaf3817cb7f21bf32cfd5495f81e353a822737a", + "is_verified": false, + "line_number": 23 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "bf0b65ac0df1a8862edde59968ff2c68edddd2ab", + "is_verified": false, + "line_number": 26 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "b5447115ba5d472a32f229b470848493a75fc452", + "is_verified": false, + "line_number": 32 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "eeab9a240a6e1b5cd4b31bf76e14c04603c31a0e", + "is_verified": false, + "line_number": 35 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "3ee480ba78002544290656ed8e6e8ea9a3168d4c", + "is_verified": false, + "line_number": 41 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "de624525ee0f0bcc3ac67b93c7f3daa84e16a4c8", + "is_verified": false, + "line_number": 50 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "4f427ba6e5afa004e2886c43d0995f3a26084add", + "is_verified": false, + "line_number": 53 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "0be8484018f7fb3ce1af9d6608b7ea5ab87aac96", + "is_verified": false, + "line_number": 59 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "5413c5e1c886ce3968419c9d148d5abf5612032d", + "is_verified": false, + "line_number": 68 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "a4602ad5a2840b980a287f16e5f38935617c12b1", + "is_verified": false, + "line_number": 79 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "cc2b8e82856eada547373d940069c35e59401f95", + "is_verified": false, + "line_number": 90 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "d1aabed662657d9a22f3d2438149558d24500cf5", + "is_verified": false, + "line_number": 101 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "32ce41dfcbd2ad529b4e24746b1c237a4f7e498f", + "is_verified": false, + "line_number": 112 + }, + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/fixtures/wire-format.json", + "hashed_secret": "2d8a7cd93348976db13c8eceab526cb7ff33fe5e", + "is_verified": false, + "line_number": 123 + } + ], + "tests/unit/protocol/test_cache_key_vectors.py": [ + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/test_cache_key_vectors.py", + "hashed_secret": "b38d3dec5d8d971e362e3a909295bfbc6c09620e", + "is_verified": false, + "line_number": 33 + } + ], + "tests/unit/protocol/test_envelope_wire_vectors.py": [ + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/test_envelope_wire_vectors.py", + "hashed_secret": "bf356d2af94fc23cf2620f7a20bb234e2db3f79f", + "is_verified": false, + "line_number": 51 + } + ], + "tests/unit/protocol/test_interop_vectors.py": [ + { + "type": "Hex High Entropy String", + "filename": "tests/unit/protocol/test_interop_vectors.py", + "hashed_secret": "92763d3c823f0972f6c57907543e0d38bfcf8f3a", + "is_verified": false, + "line_number": 39 + } + ], "tests/unit/test_structured_logging.py": [ { "type": "Base64 High Entropy String", @@ -425,5 +887,5 @@ } ] }, - "generated_at": "2026-07-25T12:35:03Z" + "generated_at": "2026-07-29T00:15:05Z" } diff --git a/Cargo.lock b/Cargo.lock index bfbc0e5..47f9414 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,9 +245,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cachekit-core" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee6235f73aefb0dc66b9cd0d333b9da928c8192e4234357a739fe756c2f8f23" +checksum = "6aba1513135a7b92a124ad6983f7e80e5f5c78c4f9c74384079efa3fbf491eab" dependencies = [ "aes", "aes-gcm", diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index cbee65d..21afe8e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -215,7 +215,20 @@ fn compress_bytes(data: &[u8]) -> PyResult> { |:----------|:---------------| | Compression | LZ4 (fast, ~3:1 ratio) | | Integrity | xxHash3-64 checksums | -| Envelope | `[version:1][format:1][checksum:8][compressed_data]` | +| Envelope | `rmp_serde` MessagePack positional `fixarray(4)`: `[compressed_data, checksum, original_size, format]` | + +The envelope has no version field or discriminator — the MessagePack marker on +element `[0]` is self-describing. Since cachekit-core 0.4.0 (protocol 1.1), +`compressed_data` encodes as MessagePack `bin` (`0xc4`/`0xc5`/`0xc6`); envelopes +written before 0.4.0 encoded it as an array of integers and remain readable +forever (dual-read is proven in both directions). `checksum` stays an array of +8 integers by deliberate spec exclusion. Normative rules and rationale: +[protocol `spec/wire-format.md`][protocol-wire] and +[`decisions/envelope-bin-encoding.md`][protocol-bin-decision]; byte-verified +locally in `tests/unit/protocol/test_envelope_wire_vectors.py`. + +[protocol-wire]: https://github.com/cachekit-io/protocol/blob/main/spec/wire-format.md +[protocol-bin-decision]: https://github.com/cachekit-io/protocol/blob/main/decisions/envelope-bin-encoding.md ### Backend Architecture diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b755275..2bc3300 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -20,7 +20,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # Compression, checksums, encryption (https://crates.io/crates/cachekit-core) -cachekit-core = { version = "0.3.0", features = ["compression", "checksum", "messagepack", "encryption"] } +cachekit-core = { version = "0.4.0", features = ["compression", "checksum", "messagepack", "encryption"] } # Python integration - optional for Rust-only builds pyo3 = { workspace = true, optional = true } diff --git a/tests/unit/protocol/fixtures/wire-format.json b/tests/unit/protocol/fixtures/wire-format.json new file mode 100644 index 0000000..9be2ba0 --- /dev/null +++ b/tests/unit/protocol/fixtures/wire-format.json @@ -0,0 +1,134 @@ +{ + "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input", + "compression": "LZ4 block format (NOT frame)", + "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]. Vectors without an 'envelope_encoding' field use the legacy array-of-integers encoding for compressed_data and are retained forever as legacy-read proof; their '*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). checksum always encodes as an array of 8 integers. Normative rules: spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'.", + "generator": "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)", + "limits": { + "max_compressed_size": 536870912, + "max_compression_ratio": 1000, + "max_uncompressed_size": 536870912 + }, + "vectors": [ + { + "description": "Zero-length payload", + "envelope_hex": "949100982d06cc800538ccd3cc94ccc200a76d73677061636b", + "envelope_size": 25, + "format": "msgpack", + "input_hex": "", + "input_size": 0, + "name": "empty" + }, + { + "description": "Short ASCII text", + "envelope_hex": "94dc0012ccf00168656c6c6f2c2063616368656b697421986dcc8a34ccb63a3c52ccd310a76d73677061636b", + "envelope_size": 44, + "format": "msgpack", + "input_hex": "68656c6c6f2c2063616368656b697421", + "input_size": 16, + "name": "simple_string" + }, + { + "description": "64 bytes derived from pi (deterministic, low compressibility)", + "envelope_hex": "94dc0042ccf031243f6acc88cc85cca308ccd31319cc8a2e03707344cca409382229cc9f31ccd0082eccfacc98ccec4e6ccc89452821cce638ccd01377ccbe5466cccf34cce90c6cccc0ccac29ccb7ccc97c50ccdd3fcc84ccd5ccb5ccb5470917982d016e0411ccedcc9e3440a76d73677061636b", + "envelope_size": 117, + "format": "msgpack", + "input_hex": "243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917", + "input_size": 64, + "name": "binary_data" + }, + { + "description": "1024 bytes of repeated 'A' (high compression ratio)", + "envelope_hex": "949f1f410100ccffccffccffcce960414141414141984d35ccad08cce9ccd77c0dcd0400a76d73677061636b", + "envelope_size": 44, + "format": "msgpack", + "input_hex": "41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141", + "input_size": 1024, + "name": "large_compressible" + }, + { + "description": "JSON-like payload (common real-world pattern)", + "envelope_hex": "94dc004cccf03b7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d98cc87673661cc9c4bcca8204aa76d73677061636b", + "envelope_size": 102, + "format": "msgpack", + "input_hex": "7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d", + "input_size": 74, + "name": "json_like" + }, + { + "description": "Single byte 0xFF (minimum non-empty payload)", + "envelope_hex": "949210ccff98ccd6ccbcccec3c6b29ccd72e01a76d73677061636b", + "envelope_size": 27, + "format": "msgpack", + "input_hex": "ff", + "input_size": 1, + "name": "single_byte" + }, + { + "description": "bin-encoded twin of 'empty': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40100982d06cc800538ccd3cc94ccc200a76d73677061636b", + "envelope_size": 26, + "format": "msgpack", + "input_hex": "", + "input_size": 0, + "name": "empty_bin", + "envelope_encoding": "bin", + "derived_from": "empty" + }, + { + "description": "bin-encoded twin of 'simple_string': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c412f00168656c6c6f2c2063616368656b697421986dcc8a34ccb63a3c52ccd310a76d73677061636b", + "envelope_size": 42, + "format": "msgpack", + "input_hex": "68656c6c6f2c2063616368656b697421", + "input_size": 16, + "name": "simple_string_bin", + "envelope_encoding": "bin", + "derived_from": "simple_string" + }, + { + "description": "bin-encoded twin of 'binary_data': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c442f031243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917982d016e0411ccedcc9e3440a76d73677061636b", + "envelope_size": 89, + "format": "msgpack", + "input_hex": "243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917", + "input_size": 64, + "name": "binary_data_bin", + "envelope_encoding": "bin", + "derived_from": "binary_data" + }, + { + "description": "bin-encoded twin of 'large_compressible': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40f1f410100ffffffe960414141414141984d35ccad08cce9ccd77c0dcd0400a76d73677061636b", + "envelope_size": 41, + "format": "msgpack", + "input_hex": "41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141", + "input_size": 1024, + "name": "large_compressible_bin", + "envelope_encoding": "bin", + "derived_from": "large_compressible" + }, + { + "description": "bin-encoded twin of 'json_like': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c44cf03b7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d98cc87673661cc9c4bcca8204aa76d73677061636b", + "envelope_size": 100, + "format": "msgpack", + "input_hex": "7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d", + "input_size": 74, + "name": "json_like_bin", + "envelope_encoding": "bin", + "derived_from": "json_like" + }, + { + "description": "bin-encoded twin of 'single_byte': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40210ff98ccd6ccbcccec3c6b29ccd72e01a76d73677061636b", + "envelope_size": 27, + "format": "msgpack", + "input_hex": "ff", + "input_size": 1, + "name": "single_byte_bin", + "envelope_encoding": "bin", + "derived_from": "single_byte" + } + ], + "version": "1.1.0" +} diff --git a/tests/unit/protocol/test_envelope_wire_vectors.py b/tests/unit/protocol/test_envelope_wire_vectors.py new file mode 100644 index 0000000..9cfe6c5 --- /dev/null +++ b/tests/unit/protocol/test_envelope_wire_vectors.py @@ -0,0 +1,234 @@ +"""SDK-level byte-verification of the ByteStorage envelope against the protocol wire-format vectors. + +Fixture: tests/unit/protocol/fixtures/wire-format.json, vendored from +cachekit-io/protocol @ a4b392be25b4f6f6633c683d3f251a73374e727b +(sha256 d6184a945d8cb22491b5c937414fe4d01e682d084ccec9dae9526d0ffba650cb). +Regenerate ONLY by re-copying from the protocol repo — never by hand. + +The fixture is append-only (protocol 1.1, decisions/envelope-bin-encoding.md): +six legacy vectors pin the pre-0.4.0 array-of-ints encoding of +``compressed_data`` and are retained forever as legacy-read proof; their six +``*_bin`` twins pin the MessagePack ``bin`` encoding that cachekit-core 0.4.0 +writers emit. This module proves — not asserts — through the real Python +paths that: + +1. **Legacy-read**: pre-0.4.0 envelopes decode through ``ByteStorage.retrieve`` + (the exact FFI call every serializer's deserialize path makes) and through + the full decorator retrieve path. +2. **Bin-emit**: fresh writes carry ``bin``-encoded envelopes (marker + ``0xc4``/``0xc5``/``0xc6`` on element[0]) inside the CK v3 frame, observed + through the real decorator store path — including the bin16/bin32 width + tiers the protocol pins cannot cover (no legacy vector compresses > 255 B). +3. **Re-encode identity**: the 0.4.0 writer reproduces every ``*_bin`` pin + byte-identically from the vector inputs. +4. **Round-trip identity** through the full stack (store → retrieve) for + compressible and incompressible payloads. + +A failure here is a wire-format break to triage, never a fixture to silently +regenerate: envelopes are shared cross-SDK (py/ts read each other's bytes), +so a changed encoding orphans or corrupts every existing cache entry. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import msgpack +import pytest + +from cachekit import cache +from cachekit._rust_serializer import ByteStorage +from cachekit.backends.file import FileBackend, FileBackendConfig +from cachekit.key_generator import CacheKeyGenerator +from cachekit.serializers.standard_serializer import StandardSerializer +from cachekit.serializers.wrapper import SerializationWrapper + +pytestmark = pytest.mark.unit + +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "wire-format.json" +FIXTURE_SHA256 = "d6184a945d8cb22491b5c937414fe4d01e682d084ccec9dae9526d0ffba650cb" + +_FIXTURE = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) +VECTORS = _FIXTURE["vectors"] +LEGACY_VECTORS = [v for v in VECTORS if "envelope_encoding" not in v] +BIN_VECTORS = [v for v in VECTORS if v.get("envelope_encoding") == "bin"] + +# MessagePack bin-family markers: bin8 / bin16 / bin32. +BIN_MARKERS = {0xC4, 0xC5, 0xC6} +# Envelope is a positional fixarray(4): [compressed_data, checksum, original_size, format]. +FIXARRAY_4 = 0x94 + + +def _incompressible(n: int) -> bytes: + """Deterministic high-entropy bytes (SHA-256 counter stream) — LZ4 cannot shrink these.""" + out = bytearray() + counter = 0 + while len(out) < n: + out += hashlib.sha256(counter.to_bytes(8, "big")).digest() + counter += 1 + return bytes(out[:n]) + + +def _to_legacy_encoding(envelope: bytes) -> bytes: + """Transcode a bin-encoded envelope to the pre-0.4.0 array-of-ints encoding. + + Round-trips the envelope through msgpack with ``compressed_data`` as a list + of ints — exactly the shape rmp_serde emitted for a plain ``Vec`` + before core 0.4.0. Element order and all other fields are untouched. + """ + compressed, checksum, original_size, fmt = msgpack.unpackb(envelope, use_list=True, raw=False) + assert isinstance(compressed, bytes), "expected a bin-encoded envelope to transcode" + legacy = msgpack.packb([list(compressed), checksum, original_size, fmt], use_bin_type=True) + assert isinstance(legacy, bytes) + return legacy + + +class TestWireFormatFixture: + """The vendored fixture is byte-identical to the pinned protocol revision.""" + + def test_fixture_integrity(self): + digest = hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest() + assert digest == FIXTURE_SHA256, ( + f"fixtures/wire-format.json sha256 {digest} != pinned {FIXTURE_SHA256}. " + "If the protocol vectors were intentionally updated, refresh the pin AND the counts." + ) + + def test_vector_counts(self): + # Append-only contract: 6 legacy vectors retained forever + 6 *_bin twins. + assert len(LEGACY_VECTORS) == 6 + assert len(BIN_VECTORS) == 6 + assert {v["derived_from"] for v in BIN_VECTORS} == {v["name"] for v in LEGACY_VECTORS} + + +class TestFfiDualRead: + """Both envelope encodings decode via the real FFI retrieve path, byte-identically.""" + + @pytest.mark.parametrize("vector", LEGACY_VECTORS, ids=lambda v: v["name"]) + def test_legacy_envelope_decodes(self, vector): + """Legacy-read proof: pre-0.4.0 array-of-ints envelopes decode forever.""" + storage = ByteStorage("msgpack") + payload, fmt = storage.retrieve(bytes.fromhex(vector["envelope_hex"])) + assert bytes(payload) == bytes.fromhex(vector["input_hex"]) + assert fmt == vector["format"] + + @pytest.mark.parametrize("vector", BIN_VECTORS, ids=lambda v: v["name"]) + def test_bin_envelope_decodes(self, vector): + storage = ByteStorage("msgpack") + payload, fmt = storage.retrieve(bytes.fromhex(vector["envelope_hex"])) + assert bytes(payload) == bytes.fromhex(vector["input_hex"]) + assert fmt == vector["format"] + + @pytest.mark.parametrize("vector", BIN_VECTORS, ids=lambda v: v["name"]) + def test_store_reencodes_bin_vectors_byte_identically(self, vector): + """The 0.4.0 writer reproduces every *_bin protocol pin exactly — proven, not asserted.""" + storage = ByteStorage("msgpack") + envelope = bytes(storage.store(bytes.fromhex(vector["input_hex"]), "msgpack")) + assert envelope.hex() == vector["envelope_hex"] + + +class TestBinEmitWidths: + """Fresh writes emit bin envelopes at every header width through the real serializer path. + + The protocol *_bin vectors only exercise bin8 (no legacy vector compresses + > 255 B), so the width tiers are pinned here against the real writer, per + the decision record's deferred-coverage note. + """ + + @pytest.mark.parametrize( + ("payload_size", "expected_marker"), + [ + (16, 0xC4), # bin8: compressed_data <= 255 B + (1024, 0xC5), # bin16: 256 B <= compressed_data <= 65535 B + (128 * 1024, 0xC6), # bin32: compressed_data > 65535 B + ], + ids=["bin8", "bin16", "bin32"], + ) + def test_serialize_emits_expected_bin_width(self, payload_size, expected_marker): + envelope, _ = StandardSerializer().serialize(_incompressible(payload_size)) + assert envelope[0] == FIXARRAY_4 + assert envelope[1] == expected_marker + compressed, checksum, original_size, fmt = msgpack.unpackb(envelope, use_list=True, raw=False) + assert isinstance(compressed, bytes) # bin decodes to bytes; legacy array-of-ints would be a list + assert isinstance(checksum, list) and len(checksum) == 8 # checksum stays array-of-ints (spec exclusion) + assert fmt == "msgpack" + + +class TestFullStackEnvelope: + """Envelope encoding proven through the real decorator store/retrieve stack (L2 = FileBackend).""" + + @pytest.fixture + def file_backend(self, tmp_path): + return FileBackend(FileBackendConfig(cache_dir=tmp_path, max_size_mb=256)) + + @staticmethod + def _stored_frame(backend, func, args, namespace): + cache_key = CacheKeyGenerator().generate_key(func, args, {}, namespace) + raw = backend.get(cache_key) + assert raw is not None, f"no stored frame for key {cache_key}" + return cache_key, bytes(raw) + + def test_fresh_write_emits_bin_envelope_through_store_path(self, file_backend): + """A fresh decorator write stores a CK v3 frame whose payload is a bin-encoded envelope.""" + + @cache(backend=file_backend, ttl=300, namespace="test_envelope_bin_emit", l1_enabled=False) + def compute(x: int) -> dict: + return {"user_id": x, "name": "Alice", "active": True} + + compute(1) + _, frame = self._stored_frame(file_backend, compute, (1,), "test_envelope_bin_emit") + + assert frame[:2] == b"CK" + payload, _metadata, _name = SerializationWrapper.unwrap(frame) + envelope = bytes(payload) + assert envelope[0] == FIXARRAY_4 + assert envelope[1] in BIN_MARKERS + compressed, checksum, _original_size, fmt = msgpack.unpackb(envelope, use_list=True, raw=False) + assert isinstance(compressed, bytes) + assert isinstance(checksum, list) and len(checksum) == 8 + assert fmt == "msgpack" + + def test_legacy_envelope_decodes_through_retrieve_path(self, file_backend): + """A pre-0.4.0 (array-of-ints) envelope planted in L2 is served by the real retrieve path.""" + calls = 0 + + @cache(backend=file_backend, ttl=300, namespace="test_envelope_legacy_read", l1_enabled=False) + def compute(x: int) -> dict: + nonlocal calls + calls += 1 + return {"user_id": x, "name": "Alice", "active": True} + + expected = compute(7) + assert calls == 1 + + # Rewrite the stored frame with its envelope transcoded to the legacy encoding. + cache_key, frame = self._stored_frame(file_backend, compute, (7,), "test_envelope_legacy_read") + payload, metadata, serializer_name = SerializationWrapper.unwrap(frame) + legacy_envelope = _to_legacy_encoding(bytes(payload)) + assert legacy_envelope != bytes(payload) # the transcode actually changed the encoding + file_backend.set(cache_key, SerializationWrapper.wrap(legacy_envelope, metadata, serializer_name), ttl=300) + + # The next read must be served from the legacy envelope, not recomputed. + assert compute(7) == expected + assert calls == 1, "legacy envelope was not decoded by the retrieve path (function re-ran)" + + @pytest.mark.parametrize( + "payload", + [b"A" * 100_000, _incompressible(100_000)], + ids=["compressible", "incompressible"], + ) + def test_round_trip_identity_full_stack(self, file_backend, payload): + """store → retrieve returns byte-identical payloads for both compressibility extremes.""" + calls = 0 + + @cache(backend=file_backend, ttl=300, namespace="test_envelope_round_trip", l1_enabled=False) + def compute(tag: str) -> bytes: + nonlocal calls + calls += 1 + return payload + + assert compute("k") == payload + assert calls == 1 + assert compute("k") == payload # served from L2 through the full retrieve path + assert calls == 1 From 0a9d3722b128c5ae5c2bfaf4bf401d34467c27d0 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Wed, 29 Jul 2026 10:36:57 +1000 Subject: [PATCH 2/2] test(envelope): apply expert-panel findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/critical/test_byte_storage_error_injection.py: the final-envelope-size security check's premise died with 0.4.0 — a 500MB incompressible payload no longer inflates ~2x (bin encoding is ~1.004x), stores fine, and the expected ValueError never raised (slow-marked, so PR CI skipped the breakage). Input now sits 16 bytes under the 512MiB input cap so the LZ4 incompressible-block overhead pushes the final envelope over the limit again. randbytes chunked at 128MiB (getrandbits takes a C int). - tests/unit/protocol/test_envelope_wire_vectors.py: merged the two byte-identical decode tests into one parametrized over all 12 vectors. - DEVELOPMENT.md: 'dual-read proven in both directions' de-ambiguated — this repo's suite proves both encodings decode, not cross-SDK direction. --- DEVELOPMENT.md | 3 ++- .../test_byte_storage_error_injection.py | 21 +++++++++++-------- .../protocol/test_envelope_wire_vectors.py | 15 ++++++------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 21afe8e..c30175c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -221,7 +221,8 @@ The envelope has no version field or discriminator — the MessagePack marker on element `[0]` is self-describing. Since cachekit-core 0.4.0 (protocol 1.1), `compressed_data` encodes as MessagePack `bin` (`0xc4`/`0xc5`/`0xc6`); envelopes written before 0.4.0 encoded it as an array of integers and remain readable -forever (dual-read is proven in both directions). `checksum` stays an array of +forever (both encodings are byte-verified against the protocol vectors in +`tests/unit/protocol/test_envelope_wire_vectors.py`). `checksum` stays an array of 8 integers by deliberate spec exclusion. Normative rules and rationale: [protocol `spec/wire-format.md`][protocol-wire] and [`decisions/envelope-bin-encoding.md`][protocol-bin-decision]; byte-verified diff --git a/tests/critical/test_byte_storage_error_injection.py b/tests/critical/test_byte_storage_error_injection.py index 4984966..33b481d 100644 --- a/tests/critical/test_byte_storage_error_injection.py +++ b/tests/critical/test_byte_storage_error_injection.py @@ -350,23 +350,26 @@ def test_validate_method_detects_various_corruptions(self): @pytest.mark.slow def test_final_envelope_size_security_check(self): - """CRITICAL: ByteStorage must check final envelope size after serialization.""" + """CRITICAL: ByteStorage must check final envelope size after serialization. + + Since cachekit-core 0.4.0 the envelope encodes compressed_data as msgpack + bin (~1.004x payload for incompressible input, protocol 1.1), so the input + must sit just under the 512MiB input cap for the LZ4 incompressible-block + overhead to push the final envelope over the limit. (Pre-0.4.0, array-of-ints + inflation tripped this at 500MB.) + """ from cachekit._rust_serializer import ByteStorage storage = ByteStorage("msgpack") - # Create data that when serialized as envelope might exceed limits - # This is tricky - need data that compresses poorly and has large metadata - # For this test, we'll verify the check exists by using a very large input - # that would create a large envelope - - # Use data near the limit that won't compress well + # Incompressible input just under the 512MiB input cap: passes the input + # check, but the final envelope (input * ~1.0039 + header) exceeds the cap. import random random.seed(42) - near_limit_data = bytes([random.randint(0, 255) for _ in range(500 * 1024 * 1024)]) # 500MB random (incompressible) + # Chunked: randbytes() overflows past 256MiB (getrandbits takes a C int). + near_limit_data = b"".join(random.randbytes(128 * 1024 * 1024) for _ in range(4))[:-16] - # This should trigger either input limit OR final envelope limit with pytest.raises(ValueError) as exc_info: storage.store(near_limit_data, None) diff --git a/tests/unit/protocol/test_envelope_wire_vectors.py b/tests/unit/protocol/test_envelope_wire_vectors.py index 9cfe6c5..7cf583b 100644 --- a/tests/unit/protocol/test_envelope_wire_vectors.py +++ b/tests/unit/protocol/test_envelope_wire_vectors.py @@ -105,16 +105,13 @@ def test_vector_counts(self): class TestFfiDualRead: """Both envelope encodings decode via the real FFI retrieve path, byte-identically.""" - @pytest.mark.parametrize("vector", LEGACY_VECTORS, ids=lambda v: v["name"]) - def test_legacy_envelope_decodes(self, vector): - """Legacy-read proof: pre-0.4.0 array-of-ints envelopes decode forever.""" - storage = ByteStorage("msgpack") - payload, fmt = storage.retrieve(bytes.fromhex(vector["envelope_hex"])) - assert bytes(payload) == bytes.fromhex(vector["input_hex"]) - assert fmt == vector["format"] + @pytest.mark.parametrize("vector", VECTORS, ids=lambda v: v["name"]) + def test_envelope_decodes(self, vector): + """Dual-read proof: legacy (array-of-ints) AND bin envelopes decode, byte-identically. - @pytest.mark.parametrize("vector", BIN_VECTORS, ids=lambda v: v["name"]) - def test_bin_envelope_decodes(self, vector): + The plain-named vectors pin the pre-0.4.0 legacy encoding, which stays a + permanently accepted read format; their ``*_bin`` twins pin the 0.4.0 writer. + """ storage = ByteStorage("msgpack") payload, fmt = storage.retrieve(bytes.fromhex(vector["envelope_hex"])) assert bytes(payload) == bytes.fromhex(vector["input_hex"])