diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..809e6bf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Vendored protocol test vectors are byte-exact artifacts, sha256-pinned by +# tests/wire_format_vectors.rs — never let git rewrite their line endings +# (windows runners set core.autocrlf=true and would break the pin). +tests/vectors/* -text diff --git a/Cargo.toml b/Cargo.toml index bae9c97..bf9a2c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,9 @@ proptest = "1.4" serde_json = "1.0" blake2 = "0.10" hex = "0.4" +# Unconditional for tests: the optional sha2 above only exists under `encryption`, +# but the wire-format vector fixture pin must verify under every feature set. +sha2 = "0.10" aes-gcm = { version = "0.10", features = ["zeroize"] } criterion = { version = "0.5", features = ["html_reports"] } diff --git a/README.md b/README.md index cad4290..6161eef 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,17 @@ cd fuzz && cargo fuzz run byte_storage_corrupted_envelope See [`fuzz/README.md`](fuzz/README.md) for comprehensive fuzzing documentation. +### Protocol wire-format vectors + +`tests/wire_format_vectors.rs` byte-verifies this crate against the canonical +ByteStorage envelope vectors from +[`cachekit-io/protocol`](https://github.com/cachekit-io/protocol) +(`test-vectors/wire-format.json`): every vector must decode to the exact +payload bytes and re-encode to the exact envelope bytes. The fixture is +vendored at `tests/vectors/wire-format.json` and integrity-pinned by sha256 — +to update it, re-copy from the protocol repo and change the pinned hash in the +same commit. + --- ## Minimum Supported Rust Version diff --git a/tests/compatibility/test_vector.json b/tests/compatibility/test_vector.json deleted file mode 100644 index 8712962..0000000 --- a/tests/compatibility/test_vector.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "original_data": "48656c6c6f2c20746869732069732074657374206461746120666f7220636f6d7061746962696c69747920766572696669636174696f6e2e", - "original_data_text": "Hello, this is test data for compatibility verification.", - "original_data_size": 56, - "compressed_data": "94dc003accf02948656c6c6f2c20746869732069732074657374206461746120666f7220636f6d7061746962696c69747920766572696669636174696f6e2edc0020cc80756327cc990873cca902ccd3cc9fcce879cccdccb94dccb93accb876cc8fccb9cce6cc9e5626cce214cce239cc881738a76d73677061636b", - "compressed_data_size": 124, - "format": "msgpack", - "compression_ratio": 2.214 -} \ No newline at end of file diff --git a/tests/compatibility_test.rs b/tests/compatibility_test.rs deleted file mode 100644 index 4327e5d..0000000 --- a/tests/compatibility_test.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Cross-version compatibility tests -//! -//! Verifies that data compressed by Python (using cachekit._rust_serializer) -//! can be decompressed by Rust (cachekit-core directly). -//! -//! This ensures forward/backward compatibility across version boundaries. - -use std::fs; -use std::path::Path; - -#[cfg(all(feature = "compression", feature = "checksum"))] -use cachekit_core::ByteStorage; - -/// Test vector structure matching Python generation format -#[derive(serde::Deserialize)] -struct TestVector { - original_data: String, // Hex-encoded original data - #[allow(dead_code)] - original_data_text: String, // Human-readable text (for reference) - original_data_size: usize, // Original byte size - compressed_data: String, // Hex-encoded compressed envelope - #[allow(dead_code)] - compressed_data_size: usize, // Compressed envelope byte size - format: String, // Format identifier (e.g., "msgpack") -} - -#[test] -#[ignore = "Format changed: Blake3 (32 bytes) → xxHash3-64 (8 bytes). Regenerate test vector after Python update."] -#[cfg(all(feature = "compression", feature = "checksum"))] -fn test_python_generated_data_decompresses_correctly() { - // Read test vector JSON - let test_vector_path = Path::new("tests/compatibility/test_vector.json"); - - if !test_vector_path.exists() { - eprintln!( - "Test vector not found at {}. Run: cd /Users/68824/code/27B/cachekit-workspace/cachekit && uv run python /Users/68824/code/27B/cachekit-workspace/generate_test_vectors.py", - test_vector_path.display() - ); - panic!("Test vector file missing"); - } - - let vector_json = - fs::read_to_string(test_vector_path).expect("Failed to read test vector file"); - - let vector: TestVector = - serde_json::from_str(&vector_json).expect("Failed to parse test vector JSON"); - - // Decode hex strings to bytes - let original_data = - hex::decode(&vector.original_data).expect("Failed to decode original_data hex"); - - let compressed_envelope = - hex::decode(&vector.compressed_data).expect("Failed to decode compressed_data hex"); - - // Verify expected format - assert_eq!( - vector.format, "msgpack", - "Test vector format should be msgpack" - ); - - // Initialize ByteStorage with matching format - let storage = ByteStorage::new(Some(vector.format.clone())); - - // CRITICAL TEST: Decompress Python-generated data - let decompressed_result = storage.retrieve(&compressed_envelope); - - assert!( - decompressed_result.is_ok(), - "Failed to decompress Python-generated data: {:?}", - decompressed_result.err() - ); - - let (decompressed_data, retrieved_format) = decompressed_result.unwrap(); - - // Verify decompressed data matches original - assert_eq!( - decompressed_data, - original_data, - "Decompressed data does not match original.\n\ - Original: {:?}\n\ - Decompressed: {:?}", - String::from_utf8_lossy(&original_data), - String::from_utf8_lossy(&decompressed_data) - ); - - // Verify format was preserved - assert_eq!( - retrieved_format, vector.format, - "Format mismatch: expected '{}', got '{}'", - vector.format, retrieved_format - ); - - // Verify size - assert_eq!( - decompressed_data.len(), - vector.original_data_size, - "Decompressed size does not match original" - ); - - println!( - "✓ Cross-version compatibility verified:\n \ - Original: {} bytes\n \ - Compressed: {} bytes\n \ - Decompressed: {} bytes\n \ - Format: {}", - original_data.len(), - vector.compressed_data_size, - decompressed_data.len(), - retrieved_format - ); -} - -#[test] -#[cfg(all(feature = "compression", feature = "checksum"))] -fn test_roundtrip_data_integrity() { - // Generate test data directly in Rust - let test_data = b"Hello, this is test data for compatibility verification."; - - let storage = ByteStorage::new(Some("msgpack".to_string())); - - // Store (compress) - let compressed = storage - .store(test_data, None) - .expect("Failed to store test data"); - - // Retrieve (decompress) - let (decompressed, format) = storage - .retrieve(&compressed) - .expect("Failed to retrieve test data"); - - // Verify integrity - assert_eq!( - &decompressed[..], - test_data, - "Roundtrip data integrity check failed" - ); - - assert_eq!(format, "msgpack", "Format mismatch in roundtrip test"); - - println!( - "✓ Roundtrip integrity verified:\n \ - Original: {} bytes\n \ - Compressed: {} bytes", - test_data.len(), - compressed.len() - ); -} - -#[test] -#[cfg(all(feature = "compression", feature = "checksum"))] -fn test_format_preservation() { - // Test that format string is preserved through compression/decompression - - let storage = ByteStorage::new(Some("msgpack".to_string())); - let test_data = b"test"; - - let compressed = storage.store(test_data, None).expect("Failed to store"); - - let (_, retrieved_format) = storage.retrieve(&compressed).expect("Failed to retrieve"); - - assert_eq!(retrieved_format, "msgpack"); - - println!("✓ Format preservation verified: msgpack"); -} diff --git a/tests/vectors/wire-format.json b/tests/vectors/wire-format.json new file mode 100644 index 0000000..dde964c --- /dev/null +++ b/tests/vectors/wire-format.json @@ -0,0 +1,68 @@ +{ + "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input", + "compression": "LZ4 block format (NOT frame)", + "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]; byte fields encode as msgpack arrays of integers, not bin", + "generator": "cachekit-core v0.2.0", + "limits": { + "max_compressed_size": 536870912, + "max_compression_ratio": 1000, + "max_uncompressed_size": 536870912 + }, + "vectors": [ + { + "description": "Zero-length payload", + "envelope_hex": "949100982d06cc800538ccd3cc94ccc200a76d73677061636b", + "envelope_size": 25, + "format": "msgpack", + "input_hex": "", + "input_size": 0, + "name": "empty" + }, + { + "description": "Short ASCII text", + "envelope_hex": "94dc0012ccf00168656c6c6f2c2063616368656b697421986dcc8a34ccb63a3c52ccd310a76d73677061636b", + "envelope_size": 44, + "format": "msgpack", + "input_hex": "68656c6c6f2c2063616368656b697421", + "input_size": 16, + "name": "simple_string" + }, + { + "description": "64 bytes derived from pi (deterministic, low compressibility)", + "envelope_hex": "94dc0042ccf031243f6acc88cc85cca308ccd31319cc8a2e03707344cca409382229cc9f31ccd0082eccfacc98ccec4e6ccc89452821cce638ccd01377ccbe5466cccf34cce90c6cccc0ccac29ccb7ccc97c50ccdd3fcc84ccd5ccb5ccb5470917982d016e0411ccedcc9e3440a76d73677061636b", + "envelope_size": 117, + "format": "msgpack", + "input_hex": "243f6a8885a308d313198a2e03707344a4093822299f31d0082efa98ec4e6c89452821e638d01377be5466cf34e90c6cc0ac29b7c97c50dd3f84d5b5b5470917", + "input_size": 64, + "name": "binary_data" + }, + { + "description": "1024 bytes of repeated 'A' (high compression ratio)", + "envelope_hex": "949f1f410100ccffccffccffcce960414141414141984d35ccad08cce9ccd77c0dcd0400a76d73677061636b", + "envelope_size": 44, + "format": "msgpack", + "input_hex": "41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141", + "input_size": 1024, + "name": "large_compressible" + }, + { + "description": "JSON-like payload (common real-world pattern)", + "envelope_hex": "94dc004cccf03b7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d98cc87673661cc9c4bcca8204aa76d73677061636b", + "envelope_size": 102, + "format": "msgpack", + "input_hex": "7b22757365725f6964223a31323334352c226e616d65223a22416c696365222c22656d61696c223a22616c696365406578616d706c652e636f6d222c22616374697665223a747275657d", + "input_size": 74, + "name": "json_like" + }, + { + "description": "Single byte 0xFF (minimum non-empty payload)", + "envelope_hex": "949210ccff98ccd6ccbcccec3c6b29ccd72e01a76d73677061636b", + "envelope_size": 27, + "format": "msgpack", + "input_hex": "ff", + "input_size": 1, + "name": "single_byte" + } + ], + "version": "1.0.0" +} diff --git a/tests/wire_format_vectors.rs b/tests/wire_format_vectors.rs new file mode 100644 index 0000000..c4c5297 --- /dev/null +++ b/tests/wire_format_vectors.rs @@ -0,0 +1,178 @@ +//! Canonical ByteStorage wire-format vectors. +//! +//! Verifies this crate — the canonical ByteStorage implementation — against the +//! byte-canonical envelope vectors pinned by the protocol spec +//! (`protocol/spec/wire-format.md`). Every vector must decode to the expected +//! payload bytes AND re-encode to the exact envelope bytes, so any encoding +//! regression (e.g. the positional-fixarray / int-array facts corrected in +//! protocol#11) fails CI here instead of breaking cross-version reads in +//! production. +//! +//! Fixture provenance: vendored from +//! `test-vectors/wire-format.json` +//! at commit `b0270b999bf827f8aa5fcc2d8640735196510326`, integrity-pinned by +//! sha256 below. To update: copy the file from a newer protocol ref, update +//! `FIXTURE_SHA256` and this comment's commit hash together. + +#![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] + +use cachekit_core::{ByteStorage, StorageEnvelope}; +use sha2::{Digest, Sha256}; + +/// Compiled-in fixture: no runtime path resolution, so the test can never be +/// silently skipped by a missing file. +const FIXTURE: &str = include_str!("vectors/wire-format.json"); + +/// sha256 of the vendored fixture — must match the protocol repo's copy. +const FIXTURE_SHA256: &str = "f83a8a18c5067c6353b7c13f0a47b4e5cefec11f4ab881b12079483174e6e0b8"; + +#[derive(serde::Deserialize)] +struct WireFormatFixture { + limits: Limits, + vectors: Vec, + version: String, +} + +#[derive(serde::Deserialize)] +struct Limits { + max_compressed_size: usize, + max_compression_ratio: u64, + max_uncompressed_size: usize, +} + +#[derive(serde::Deserialize)] +struct Vector { + name: String, + format: String, + input_hex: String, + input_size: usize, + envelope_hex: String, + envelope_size: usize, +} + +fn load_fixture() -> WireFormatFixture { + serde_json::from_str(FIXTURE).expect("wire-format.json fixture must parse") +} + +#[test] +fn fixture_integrity_pinned_sha256() { + let digest = hex::encode(Sha256::digest(FIXTURE.as_bytes())); + assert_eq!( + digest, FIXTURE_SHA256, + "vendored wire-format.json drifted from its pinned sha256 — \ + re-vendor from the protocol repo and update FIXTURE_SHA256 deliberately" + ); +} + +#[test] +fn fixture_is_current_version_with_vectors() { + let fixture = load_fixture(); + assert_eq!(fixture.version, "1.0.0"); + assert!( + fixture.vectors.len() >= 6, + "vector set is append-only; expected at least the original 6, got {}", + fixture.vectors.len() + ); +} + +#[test] +fn fixture_limits_match_implementation() { + let fixture = load_fixture(); + let storage = ByteStorage::new(None); + assert_eq!( + fixture.limits.max_uncompressed_size, + storage.max_uncompressed_size() + ); + assert_eq!( + fixture.limits.max_compressed_size, + storage.max_compressed_size() + ); + assert_eq!( + fixture.limits.max_compression_ratio, + storage.max_compression_ratio() + ); +} + +/// Decode direction: every canonical envelope must retrieve to the exact +/// original payload bytes, format, and size. +#[test] +fn vectors_decode_to_expected_payload() { + let storage = ByteStorage::new(None); + for vector in load_fixture().vectors { + let input = hex::decode(&vector.input_hex).expect("input_hex must decode"); + let envelope = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); + assert_eq!( + input.len(), + vector.input_size, + "[{}] input_size mismatch", + vector.name + ); + assert_eq!( + envelope.len(), + vector.envelope_size, + "[{}] envelope_size mismatch", + vector.name + ); + + let (payload, format) = storage + .retrieve(&envelope) + .unwrap_or_else(|e| panic!("[{}] retrieve failed: {e:?}", vector.name)); + assert_eq!( + payload, input, + "[{}] decoded payload differs from input", + vector.name + ); + assert_eq!(format, vector.format, "[{}] format mismatch", vector.name); + assert!( + storage.validate(&envelope), + "[{}] validate() rejected canonical envelope", + vector.name + ); + } +} + +/// Encode direction: storing the original payload must reproduce the exact +/// envelope bytes. The full path (LZ4 block encoding + positional-array +/// MessagePack) is deterministic; a failure here means the wire bytes changed +/// and cross-version reads are at risk — regenerate vectors deliberately in +/// the protocol repo, never adjust expectations here. +#[test] +fn vectors_reencode_byte_identical() { + let storage = ByteStorage::new(None); + for vector in load_fixture().vectors { + let input = hex::decode(&vector.input_hex).expect("input_hex must decode"); + let expected = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); + + let encoded = storage + .store(&input, Some(vector.format.clone())) + .unwrap_or_else(|e| panic!("[{}] store failed: {e:?}", vector.name)); + assert_eq!( + hex::encode(&encoded), + hex::encode(&expected), + "[{}] re-encoded envelope is not byte-identical to the canonical vector", + vector.name + ); + } +} + +/// Envelope-codec identity, independent of LZ4: deserializing the canonical +/// bytes into StorageEnvelope and re-serializing must be byte-identical. When +/// `vectors_reencode_byte_identical` fails, this localizes the regression — +/// codec test failing too means the MessagePack layout changed (protocol#11 +/// territory); codec test passing means the LZ4 block encoding changed. +#[test] +fn envelope_codec_roundtrip_byte_identical() { + for vector in load_fixture().vectors { + let canonical = hex::decode(&vector.envelope_hex).expect("envelope_hex must decode"); + let envelope: StorageEnvelope = rmp_serde::from_slice(&canonical) + .unwrap_or_else(|e| panic!("[{}] envelope must deserialize: {e}", vector.name)); + let reserialized = rmp_serde::to_vec(&envelope) + .unwrap_or_else(|e| panic!("[{}] envelope must reserialize: {e}", vector.name)); + assert_eq!( + hex::encode(&reserialized), + hex::encode(&canonical), + "[{}] MessagePack envelope layout is not byte-stable", + vector.name + ); + } +}