diff --git a/.secrets.baseline b/.secrets.baseline index aea60f0..04bab19 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -557,14 +557,21 @@ "filename": "packages/cachekit/test/protocol/wire-format.protocol.test.ts", "hashed_secret": "94a0e90cfabf50717f7f969ff1b8d5335f2b0ed2", "is_verified": false, - "line_number": 135 + "line_number": 157 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/protocol/wire-format.protocol.test.ts", + "hashed_secret": "2b21ce9d4e7ce00f8be9e9cb2634b5e01c7d91d9", + "is_verified": false, + "line_number": 160 }, { "type": "Hex High Entropy String", "filename": "packages/cachekit/test/protocol/wire-format.protocol.test.ts", "hashed_secret": "433223517d64226aefd32d9b2e6bebaa8599bddd", "is_verified": false, - "line_number": 137 + "line_number": 165 } ], "packages/cachekit/test/workers/fixtures/encryption.json": [ @@ -737,8 +744,50 @@ "hashed_secret": "0be8484018f7fb3ce1af9d6608b7ea5ab87aac96", "is_verified": false, "line_number": 59 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/workers/fixtures/wire-format.json", + "hashed_secret": "5413c5e1c886ce3968419c9d148d5abf5612032d", + "is_verified": false, + "line_number": 68 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/workers/fixtures/wire-format.json", + "hashed_secret": "a4602ad5a2840b980a287f16e5f38935617c12b1", + "is_verified": false, + "line_number": 79 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/workers/fixtures/wire-format.json", + "hashed_secret": "cc2b8e82856eada547373d940069c35e59401f95", + "is_verified": false, + "line_number": 90 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/workers/fixtures/wire-format.json", + "hashed_secret": "d1aabed662657d9a22f3d2438149558d24500cf5", + "is_verified": false, + "line_number": 101 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/workers/fixtures/wire-format.json", + "hashed_secret": "32ce41dfcbd2ad529b4e24746b1c237a4f7e498f", + "is_verified": false, + "line_number": 112 + }, + { + "type": "Hex High Entropy String", + "filename": "packages/cachekit/test/workers/fixtures/wire-format.json", + "hashed_secret": "2d8a7cd93348976db13c8eceab526cb7ff33fe5e", + "is_verified": false, + "line_number": 123 } ] }, - "generated_at": "2026-07-25T14:27:30Z" + "generated_at": "2026-07-29T00:08:01Z" } diff --git a/packages/cachekit-core-ts/Cargo.lock b/packages/cachekit-core-ts/Cargo.lock index f72cd55..2ca505b 100644 --- a/packages/cachekit-core-ts/Cargo.lock +++ b/packages/cachekit-core-ts/Cargo.lock @@ -130,9 +130,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cachekit-core" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee6235f73aefb0dc66b9cd0d333b9da928c8192e4234357a739fe756c2f8f23" +checksum = "6aba1513135a7b92a124ad6983f7e80e5f5c78c4f9c74384079efa3fbf491eab" dependencies = [ "aes", "aes-gcm", diff --git a/packages/cachekit-core-ts/Cargo.toml b/packages/cachekit-core-ts/Cargo.toml index e0c9e3d..087ce08 100644 --- a/packages/cachekit-core-ts/Cargo.toml +++ b/packages/cachekit-core-ts/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [dependencies] napi = { version = "3", features = ["napi6"] } napi-derive = "3" -cachekit-core = { version = "0.3.0", features = ["encryption"] } +cachekit-core = { version = "0.4.0", features = ["encryption"] } [build-dependencies] napi-build = "2" diff --git a/packages/cachekit-core-wasm/Cargo.lock b/packages/cachekit-core-wasm/Cargo.lock index bc4b697..a33fc20 100644 --- a/packages/cachekit-core-wasm/Cargo.lock +++ b/packages/cachekit-core-wasm/Cargo.lock @@ -130,9 +130,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cachekit-core" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee6235f73aefb0dc66b9cd0d333b9da928c8192e4234357a739fe756c2f8f23" +checksum = "6aba1513135a7b92a124ad6983f7e80e5f5c78c4f9c74384079efa3fbf491eab" dependencies = [ "aes", "aes-gcm", diff --git a/packages/cachekit-core-wasm/Cargo.toml b/packages/cachekit-core-wasm/Cargo.toml index 86a66ab..004cd1d 100644 --- a/packages/cachekit-core-wasm/Cargo.toml +++ b/packages/cachekit-core-wasm/Cargo.toml @@ -18,7 +18,7 @@ crate-type = ["cdylib"] [dependencies] wasm-bindgen = "=0.2.121" -cachekit-core = { version = "0.3.0", features = ["encryption"] } +cachekit-core = { version = "0.4.0", features = ["encryption"] } # Standalone crate — never join an enclosing cargo workspace. [workspace] diff --git a/packages/cachekit/test/protocol/wire-format.protocol.test.ts b/packages/cachekit/test/protocol/wire-format.protocol.test.ts index 067f605..d9bf55c 100644 --- a/packages/cachekit/test/protocol/wire-format.protocol.test.ts +++ b/packages/cachekit/test/protocol/wire-format.protocol.test.ts @@ -1,5 +1,27 @@ import { describe, it, expect } from 'vitest'; import { ByteStorage } from '@cachekit-io/cachekit-core-ts'; +// Single vendored copy of protocol/test-vectors/wire-format.json (see the +// workers lane header for the re-copy rule); this lane runs the same vectors +// through the NAPI binding so both bindings are held to identical bytes. +import fixture from '../workers/fixtures/wire-format.json'; + +interface WireVector { + name: string; + description: string; + input_hex: string; + envelope_hex: string; + format: string; + /** "bin" on protocol 1.1 vectors; absent on legacy array-of-integers vectors. */ + envelope_encoding?: string; +} + +const vectors = fixture.vectors as WireVector[]; +const binVectors = vectors.filter((v) => v.envelope_encoding === 'bin'); +const legacyVectors = vectors.filter((v) => v.envelope_encoding === undefined); + +// msgpack bin format markers: bin8 / bin16 / bin32. The envelope is a +// 4-element fixarray (0x94), so byte 1 is the first byte of compressed_data. +const MSGPACK_BIN_MARKERS = [0xc4, 0xc5, 0xc6]; function hexToBytes(hex: string): Uint8Array { const bytes = new Uint8Array(hex.length / 2); @@ -16,12 +38,14 @@ function bytesToHex(bytes: Uint8Array): string { } /** - * Protocol v1.0 Wire Format (ByteStorage Envelope) Tests + * Protocol v1.1 Wire Format (ByteStorage Envelope) Tests * * Verifies LZ4 compression + xxHash3-64 integrity wrapping via the Rust NAPI binding. * The envelope format is a MessagePack array: [compressed_data, checksum, original_size, format]. + * Since protocol 1.1 (cachekit-core 0.4.0) fresh packs encode compressed_data as msgpack + * bin; legacy array-of-integers envelopes stay readable forever (dual-read). */ -describe('Protocol v1.0 Wire Format (ByteStorage)', () => { +describe('Protocol v1.1 Wire Format (ByteStorage)', () => { const bs = new ByteStorage(); describe('pack/unpack round-trip', () => { @@ -130,13 +154,17 @@ describe('Protocol v1.0 Wire Format (ByteStorage)', () => { }); describe('canonical cross-SDK fixture', () => { - // Generated from Python: ByteStorage("msgpack").store(b"Hello, CacheKit!", "msgpack") - // Verified byte-for-byte identical output from both Python and TypeScript NAPI. const CANONICAL_INPUT_HEX = '48656c6c6f2c2043616368654b697421'; // "Hello, CacheKit!" + // Protocol 1.1 (core 0.4.0): compressed_data as msgpack bin. const CANONICAL_PACKED_HEX = + '94c412f00148656c6c6f2c2043616368654b69742198796ecced283c5d69cc8d10a76d73677061636b'; + // Pre-0.4.0 encoding of the same envelope (array-of-integers, generated + // from Python: ByteStorage("msgpack").store(b"Hello, CacheKit!", "msgpack")). + // Kept forever: protocol 1.1 is dual-read. + const LEGACY_PACKED_HEX = '94dc0012ccf00148656c6c6f2c2043616368654b69742198796ecced283c5d69cc8d10a76d73677061636b'; - it('pack produces canonical envelope byte-for-byte', () => { + it('pack produces canonical bin envelope byte-for-byte', () => { const input = hexToBytes(CANONICAL_INPUT_HEX); const packed = bs.pack(input); expect(bytesToHex(packed)).toBe(CANONICAL_PACKED_HEX); @@ -148,12 +176,50 @@ describe('Protocol v1.0 Wire Format (ByteStorage)', () => { expect(bytesToHex(unpacked)).toBe(CANONICAL_INPUT_HEX); }); - it('Python-generated envelope unpacks correctly in TypeScript', () => { - // This is the critical cross-SDK interop assertion: - // Python packed this → TypeScript must unpack it identically - const pythonPacked = hexToBytes(CANONICAL_PACKED_HEX); - const result = bs.unpack(pythonPacked); + it('legacy (pre-0.4.0) envelope unpacks correctly', () => { + // The critical mixed-version assertion: values packed by any pre-bin + // writer must keep decoding identically. + const legacyPacked = hexToBytes(LEGACY_PACKED_HEX); + const result = bs.unpack(legacyPacked); expect(bytesToHex(result)).toBe(CANONICAL_INPUT_HEX); }); }); + + describe('protocol wire-format.json vectors', () => { + it.each(binVectors.map((v) => [v.name, v] as const))( + 'pack emits the protocol 1.1 bin envelope byte-for-byte (%s)', + (_name, vector) => { + const packed = bs.pack(hexToBytes(vector.input_hex)); + expect(bytesToHex(packed)).toBe(vector.envelope_hex); + } + ); + + it.each(vectors.map((v) => [v.name, v] as const))( + 'unpacks ground-truth envelope %s', + (_name, vector) => { + const unpacked = bs.unpack(hexToBytes(vector.envelope_hex)); + expect(bytesToHex(unpacked)).toBe(vector.input_hex); + } + ); + + it('decodes every legacy (pre-bin) envelope', () => { + expect(legacyVectors.length).toBeGreaterThan(0); + for (const vector of legacyVectors) { + expect(bytesToHex(bs.unpack(hexToBytes(vector.envelope_hex)))).toBe(vector.input_hex); + } + }); + + it('pack marks compressed_data as msgpack bin for arbitrary payloads', () => { + // Not in the vector set: one bin8-sized and one bin16-sized payload. + const small = new TextEncoder().encode('fresh bin-emit check'); + const large = new Uint8Array(1000); + for (let i = 0; i < large.length; i++) large[i] = (i * 131 + 17) & 0xff; // incompressible + for (const payload of [small, large]) { + const packed = bs.pack(payload); + expect(packed[0]).toBe(0x94); // fixarray(4) envelope + expect(MSGPACK_BIN_MARKERS).toContain(packed[1]); + expect(bs.unpack(packed)).toEqual(payload); + } + }); + }); }); diff --git a/packages/cachekit/test/workers/fixtures/wire-format.json b/packages/cachekit/test/workers/fixtures/wire-format.json index dde964c..9be2ba0 100644 --- a/packages/cachekit/test/workers/fixtures/wire-format.json +++ b/packages/cachekit/test/workers/fixtures/wire-format.json @@ -1,8 +1,8 @@ { "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input", "compression": "LZ4 block format (NOT frame)", - "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]; byte fields encode as msgpack arrays of integers, not bin", - "generator": "cachekit-core v0.2.0", + "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]. Vectors without an 'envelope_encoding' field use the legacy array-of-integers encoding for compressed_data and are retained forever as legacy-read proof; their '*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). checksum always encodes as an array of 8 integers. Normative rules: spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'.", + "generator": "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)", "limits": { "max_compressed_size": 536870912, "max_compression_ratio": 1000, @@ -62,7 +62,73 @@ "input_hex": "ff", "input_size": 1, "name": "single_byte" + }, + { + "description": "bin-encoded twin of 'empty': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40100982d06cc800538ccd3cc94ccc200a76d73677061636b", + "envelope_size": 26, + "format": "msgpack", + "input_hex": "", + "input_size": 0, + "name": "empty_bin", + "envelope_encoding": "bin", + "derived_from": "empty" + }, + { + "description": "bin-encoded twin of 'simple_string': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c412f00168656c6c6f2c2063616368656b697421986dcc8a34ccb63a3c52ccd310a76d73677061636b", + "envelope_size": 42, + "format": "msgpack", + "input_hex": "68656c6c6f2c2063616368656b697421", + "input_size": 16, + "name": "simple_string_bin", + "envelope_encoding": "bin", + "derived_from": "simple_string" + }, + { + "description": "bin-encoded twin of 'binary_data': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c442f031243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917982d016e0411ccedcc9e3440a76d73677061636b", + "envelope_size": 89, + "format": "msgpack", + "input_hex": "243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917", + "input_size": 64, + "name": "binary_data_bin", + "envelope_encoding": "bin", + "derived_from": "binary_data" + }, + { + "description": "bin-encoded twin of 'large_compressible': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40f1f410100ffffffe960414141414141984d35ccad08cce9ccd77c0dcd0400a76d73677061636b", + "envelope_size": 41, + "format": "msgpack", + "input_hex": "41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141", + "input_size": 1024, + "name": "large_compressible_bin", + "envelope_encoding": "bin", + "derived_from": "large_compressible" + }, + { + "description": "bin-encoded twin of 'json_like': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c44cf03b7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d98cc87673661cc9c4bcca8204aa76d73677061636b", + "envelope_size": 100, + "format": "msgpack", + "input_hex": "7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d", + "input_size": 74, + "name": "json_like_bin", + "envelope_encoding": "bin", + "derived_from": "json_like" + }, + { + "description": "bin-encoded twin of 'single_byte': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c40210ff98ccd6ccbcccec3c6b29ccd72e01a76d73677061636b", + "envelope_size": 27, + "format": "msgpack", + "input_hex": "ff", + "input_size": 1, + "name": "single_byte_bin", + "envelope_encoding": "bin", + "derived_from": "single_byte" } ], - "version": "1.0.0" + "version": "1.1.0" } diff --git a/packages/cachekit/test/workers/wire-format.workers.test.ts b/packages/cachekit/test/workers/wire-format.workers.test.ts index b8b6528..ea36b53 100644 --- a/packages/cachekit/test/workers/wire-format.workers.test.ts +++ b/packages/cachekit/test/workers/wire-format.workers.test.ts @@ -18,9 +18,17 @@ interface WireVector { input_hex: string; envelope_hex: string; format: string; + /** "bin" on protocol 1.1 vectors; absent on legacy array-of-integers vectors. */ + envelope_encoding?: string; } const vectors = fixture.vectors as WireVector[]; +const binVectors = vectors.filter((v) => v.envelope_encoding === 'bin'); +const legacyVectors = vectors.filter((v) => v.envelope_encoding === undefined); + +// msgpack bin format markers: bin8 / bin16 / bin32. The envelope is a +// 4-element fixarray (0x94), so byte 1 is the first byte of compressed_data. +const MSGPACK_BIN_MARKERS = [0xc4, 0xc5, 0xc6]; function hexToBytes(hex: string): Uint8Array { const bytes = new Uint8Array(hex.length / 2); @@ -62,6 +70,40 @@ describe('wire-format vectors (wasm ByteStorage)', () => { } ); + // Protocol 1.1 (core 0.4.0): fresh packs emit compressed_data as msgpack + // bin. Byte-equality against the *_bin ground-truth vectors proves both the + // bin emit and byte-identity with every other SDK (the NAPI lane asserts + // the same vectors). + it.each(binVectors.map((v) => [v.name, v] as const))( + 'pack emits the protocol 1.1 bin envelope byte-for-byte (%s)', + (_name, vector) => { + const packed = storage.pack(hexToBytes(vector.input_hex)); + expect(bytesToHex(packed)).toBe(vector.envelope_hex); + } + ); + + it('pack marks compressed_data as msgpack bin for arbitrary payloads', () => { + // Not in the vector set: one bin8-sized and one bin16-sized payload. + const small = new TextEncoder().encode('fresh bin-emit check'); + const large = new Uint8Array(1000); + for (let i = 0; i < large.length; i++) large[i] = (i * 131 + 17) & 0xff; // incompressible + for (const payload of [small, large]) { + const packed = storage.pack(payload); + expect(packed[0]).toBe(0x94); // fixarray(4) envelope + expect(MSGPACK_BIN_MARKERS).toContain(packed[1]); + expect(storage.unpack(packed)).toEqual(payload); + } + }); + + // Permanent legacy read (protocol 1.1 dual-read): pre-0.4.0 array-of- + // integers envelopes must keep decoding forever. + it('decodes every legacy (pre-bin) envelope', () => { + expect(legacyVectors.length).toBeGreaterThan(0); + for (const vector of legacyVectors) { + expect(bytesToHex(storage.unpack(hexToBytes(vector.envelope_hex)))).toBe(vector.input_hex); + } + }); + it('rejects corrupted envelopes', () => { const packed = storage.pack(new TextEncoder().encode('integrity check payload')); const corrupted = packed.slice();