From 5178932f16bcfe9bc1f800cf425d5ba167ed5f7c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 12:10:26 +0000 Subject: [PATCH 01/10] fix: store UTF-8 FSON strings without 7-bit terminator corruption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Legacy fast string encoding ORs 0x80 onto the final byte and clears bit 7 on decode, so any UTF-8 payload (e.g. "café") truncates, mutates, and desynchronizes the FSON stream — bricking YarDB files after create. Keep the ASCII terminator codec for existing databases; emit length- prefixed utf8 / name_utf8 tags for high-bit content. --- xson/xson-fast.c++m | 56 +++++++++++++++++++++++++++++++++++------ xson/xson-fast.test.c++ | 19 ++++++++++++++ xson/xson-fson.c++m | 55 ++++++++++++++++++++++++++++++++++------ xson/xson-fson.test.c++ | 34 ++++++++++++++++++++++++- 4 files changed, 148 insertions(+), 16 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index e778a17..b33db64 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -113,18 +113,44 @@ inline void encode(std::ostream& os, std::int32_t i) encode(os,static_cast(i)); } +// True when every byte is in 0x00–0x7F. The legacy terminator string codec +// ORs 0x80 onto the final byte and clears bit 7 on decode, so it cannot +// round-trip UTF-8 (or any high-bit content). +inline bool is_ascii7(std::string_view str) noexcept +{ + for(const unsigned char byte : str) + { + if(byte & 0x80u) + return false; + } + return true; +} + +// Legacy 7-bit string codec (high bit marks the final byte). Only safe for +// ASCII content — callers must use encode_bytes/decode_bytes for UTF-8. inline void encode(std::ostream& os, const std::string& str) { - + if(not is_ascii7(str)) + throw std::runtime_error{ + "xson::fast legacy string encoding requires 7-bit content; use encode_bytes for UTF-8"}; + if(str.empty()) { - os.put(0x80); // Empty string marker + os.put(static_cast(0x80)); // Empty string marker return; } auto head = str.cbegin(); auto tail = head + (str.size() - 1); std::copy(head, tail, std::ostream_iterator(os)); - os.put(*tail | 0x80); + os.put(static_cast(static_cast(*tail) | 0x80u)); +} + +// Length-prefixed raw bytes (UTF-8 safe). Used by FSON utf8 / name_utf8 tags. +inline void encode_bytes(std::ostream& os, std::string_view bytes) +{ + encode(os, static_cast(bytes.size())); + if(not bytes.empty()) + os.write(bytes.data(), static_cast(bytes.size())); } inline void encode(std::ostream& os, std::double_t d) @@ -216,13 +242,29 @@ inline void decode(std::istream& is, std::int64_t& i) inline void decode(std::istream& is, std::string& str) { str.clear(); - for(char byte = is.get(); is; byte = is.get()) + for(int ch = is.get(); is; ch = is.get()) { - str.push_back(byte & 0x7f); - if(byte & 0x80) + const auto byte = static_cast(ch); + str.push_back(static_cast(byte & 0x7fu)); + if(byte & 0x80u) break; } - +} + +inline void decode_bytes(std::istream& is, std::string& str) +{ + std::uint64_t length = 0; + decode(is, length); + if(not is) + return; + + str.assign(static_cast(length), '\0'); + if(length == 0) + return; + + is.read(str.data(), static_cast(length)); + if(static_cast(is.gcount()) != length) + throw std::runtime_error{"xson::fast truncated length-prefixed string"}; } inline void decode(std::istream& is, std::double_t& d) diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index 2e962e3..bd4beea 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -86,6 +86,25 @@ auto register_tests() require_eq(s1, s2); }; + test_case("Legacy string encode rejects UTF-8, [xson]") = [] { + // High-bit bytes terminate the legacy codec early and clear bit 7 — + // refusing them here forces callers onto encode_bytes / FSON utf8. + require_throws([] { + auto ss = std::stringstream{}; + xson::fast::encode(ss, "café"s); + }); + }; + + test_case("Length-prefixed bytes round-trip UTF-8, [xson]") = [] { + auto ss = std::stringstream{}; + const auto s1 = "café 世界 🌍"s; + xson::fast::encode_bytes(ss, s1); + + auto s2 = ""s; + xson::fast::decode_bytes(ss, s2); + require_eq(s1, s2); + }; + return 0; } diff --git a/xson/xson-fson.c++m b/xson/xson-fson.c++m index 20a00e1..1ececb4 100644 --- a/xson/xson-fson.c++m +++ b/xson/xson-fson.c++m @@ -28,11 +28,15 @@ enum class type : char8_t integer = '\x11', timestamp = '\x12', + // Length-prefixed UTF-8 string value (legacy `string` is 7-bit terminator). + utf8 = '\x13', // control types name = '\x1A', - end = '\x1B' + end = '\x1B', + // Length-prefixed UTF-8 object key (legacy `name` payload is 7-bit). + name_utf8 = '\x1C' }; inline auto& operator << (std::ostream& os, type t) @@ -51,6 +55,25 @@ public: void encode(std::ostream& os, const xson::object& o) { + // String payloads need a tag chosen from content: legacy `string` is a + // 7-bit terminator codec and cannot store UTF-8. Non-ASCII values use + // length-prefixed `utf8` so existing ASCII databases keep reading. + if(o.is_string()) + { + const auto& str = std::get(o.get()); + if(fast::is_ascii7(str)) + { + fast::encode(os, type::string); + fast::encode(os, str); + } + else + { + fast::encode(os, type::utf8); + fast::encode_bytes(os, str); + } + return; + } + auto type = make_type(o); fast::encode(os,type); @@ -60,9 +83,17 @@ public: case type::object: for(const auto& [name,value] : o.get()) { - fast::encode(os,type::name); // type - fast::encode(os,name); // name - encode(os,value); // object + if(fast::is_ascii7(name)) + { + fast::encode(os,type::name); + fast::encode(os,name); + } + else + { + fast::encode(os,type::name_utf8); + fast::encode_bytes(os,name); + } + encode(os,value); } fast::encode(os,type::end); break; @@ -83,10 +114,6 @@ public: fast::encode(os,std::get(o.get())); break; - case type::string: - fast::encode(os,std::get(o.get())); - break; - case type::boolean: fast::encode(os,std::get(o.get())); break; @@ -173,6 +200,13 @@ public: m_builder.name(name); break; + case type::name_utf8: + if(parent.empty() || parent.top() != type::object) + throw std::runtime_error{"Invalid FSON: name_utf8 outside object"s}; + fast::decode_bytes(is,name); + m_builder.name(name); + break; + case type::array: parent.push(type::array); m_builder.start_array(); @@ -188,6 +222,11 @@ public: m_builder.value(str); break; + case type::utf8: + fast::decode_bytes(is,str); + m_builder.value(str); + break; + case type::boolean: fast::decode(is,b); m_builder.value(b); diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index 354e21d..8afa098 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -126,7 +126,39 @@ auto register_tests() succeed(xson::json::stringify(o2, 2)); check_eq(o1["Test"s], o2["Test"s]); - check_eq(o1["Test"s].get(), o2["Test"s].get()); + check_eq(o1["Test"s].get(), o2["Test"s"].get()); + }; + + test_case("UTF-8 string value round-trip, [xson]") = [] { + // Legacy terminator string codec corrupted high bytes and desynced the + // stream; utf8 tag + length prefix must preserve document text. + auto o1 = object{ + {"name"s, "café"s}, + {"note"s, "Hello 世界 🌍"s}, + {"ascii"s, "plain"s} + }; + + auto ss = std::stringstream{}; + xson::fson::encoder{}.encode(ss, o1); + const auto o2 = xson::fson::parse(ss); + + require_eq(static_cast(o2["name"s]), "café"s); + require_eq(static_cast(o2["note"s]), "Hello 世界 🌍"s); + require_eq(static_cast(o2["ascii"s]), "plain"s); + }; + + test_case("UTF-8 object key round-trip, [xson]") = [] { + auto o1 = object{}; + o1["名"s] = "value"s; + o1["plain"s] = 1ll; + + auto ss = std::stringstream{}; + xson::fson::encoder{}.encode(ss, o1); + const auto o2 = xson::fson::parse(ss); + + require_true(o2.has("名"s)); + require_eq(static_cast(o2["名"s]), "value"s); + require_eq(static_cast(o2["plain"s]), 1ll); }; test_case("Boolean, [xson]") = [] { From e67aa9e808361a51bad101f7f0d54d1479aaeb49 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 12:12:28 +0000 Subject: [PATCH 02/10] test: fix typo in FSON ASCII string assertion --- xson/xson-fson.test.c++ | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index 8afa098..16f20f3 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -126,7 +126,7 @@ auto register_tests() succeed(xson::json::stringify(o2, 2)); check_eq(o1["Test"s], o2["Test"s]); - check_eq(o1["Test"s].get(), o2["Test"s"].get()); + check_eq(o1["Test"s].get(), o2["Test"s].get()); }; test_case("UTF-8 string value round-trip, [xson]") = [] { From 278e2c302fc5fea8a6535231995c72aab1c5e4f5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 17:58:33 +0000 Subject: [PATCH 03/10] fix: Escape/unEscape UTF-8 in FSON string and name codecs Replace encode-time utf8/name_utf8 tags with Escape/unEscape so high-bit content rides the existing string/name types through the 7-bit FAST terminator codec. Ordinary ASCII stays bit-identical on the wire; the decoder still accepts the intermediate length-prefixed utf8 tags. Co-authored-by: Kaius Ruokonen --- xson/xson-fast.c++m | 4 +- xson/xson-fast.test.c++ | 2 +- xson/xson-fson.c++m | 119 ++++++++++++++++++++++++++++------------ xson/xson-fson.test.c++ | 34 +++++++++++- 4 files changed, 120 insertions(+), 39 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index b33db64..eeef1ba 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -127,12 +127,12 @@ inline bool is_ascii7(std::string_view str) noexcept } // Legacy 7-bit string codec (high bit marks the final byte). Only safe for -// ASCII content — callers must use encode_bytes/decode_bytes for UTF-8. +// ASCII content — FSON callers must Escape first; raw UTF-8 uses encode_bytes. inline void encode(std::ostream& os, const std::string& str) { if(not is_ascii7(str)) throw std::runtime_error{ - "xson::fast legacy string encoding requires 7-bit content; use encode_bytes for UTF-8"}; + "xson::fast legacy string encoding requires 7-bit content; use Escape or encode_bytes"}; if(str.empty()) { diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index bd4beea..0c6734c 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -88,7 +88,7 @@ auto register_tests() test_case("Legacy string encode rejects UTF-8, [xson]") = [] { // High-bit bytes terminate the legacy codec early and clear bit 7 — - // refusing them here forces callers onto encode_bytes / FSON utf8. + // refusing them here forces callers onto Escape / encode_bytes. require_throws([] { auto ss = std::stringstream{}; xson::fast::encode(ss, "café"s); diff --git a/xson/xson-fson.c++m b/xson/xson-fson.c++m index 1ececb4..19834d6 100644 --- a/xson/xson-fson.c++m +++ b/xson/xson-fson.c++m @@ -28,14 +28,15 @@ enum class type : char8_t integer = '\x11', timestamp = '\x12', - // Length-prefixed UTF-8 string value (legacy `string` is 7-bit terminator). + // Legacy length-prefixed UTF-8 (superseded by Escape/unEscape on string/name). + // Still accepted on decode for documents written by the intermediate codec. utf8 = '\x13', // control types name = '\x1A', end = '\x1B', - // Length-prefixed UTF-8 object key (legacy `name` payload is 7-bit). + // Legacy length-prefixed UTF-8 object key (see utf8). name_utf8 = '\x1C' }; @@ -45,6 +46,76 @@ inline auto& operator << (std::ostream& os, type t) return os; } +// Escape marker for the 7-bit FAST string terminator codec. Chosen as a +// control byte so ordinary document text (ASCII letters, punctuation, etc.) +// keeps a bit-identical wire form; only U+0001 or high-bit UTF-8 is rewritten. +inline constexpr char escape_marker = '\x01'; + +inline constexpr int hex_value(char c) noexcept +{ + if(c >= '0' && c <= '9') return c - '0'; + if(c >= 'A' && c <= 'F') return c - 'A' + 10; + if(c >= 'a' && c <= 'f') return c - 'a' + 10; + return -1; +} + +// Rewrite bytes that the FAST stop-bit string codec cannot store (bit 7 set) +// and the escape marker itself into a 7-bit-safe form: ESC + two hex digits. +// Pure ASCII without U+0001 is unchanged, so existing databases keep working. +inline std::string Escape(std::string_view in) +{ + static constexpr char hex[] = "0123456789ABCDEF"; + + std::string out; + out.reserve(in.size()); + + for(const unsigned char byte : in) + { + if(byte == static_cast(escape_marker) || (byte & 0x80u)) + { + out.push_back(escape_marker); + out.push_back(hex[(byte >> 4) & 0x0Fu]); + out.push_back(hex[byte & 0x0Fu]); + } + else + { + out.push_back(static_cast(byte)); + } + } + + return out; +} + +// Inverse of Escape. Identity for strings that contain no escape marker. +inline std::string unEscape(std::string_view in) +{ + std::string out; + out.reserve(in.size()); + + for(std::size_t i = 0; i < in.size(); ++i) + { + const char c = in[i]; + if(c != escape_marker) + { + out.push_back(c); + continue; + } + + if(i + 2 >= in.size()) + throw std::runtime_error{"Invalid FSON: truncated escape sequence"s}; + + const int hi = hex_value(in[i + 1]); + const int lo = hex_value(in[i + 2]); + if(hi < 0 || lo < 0) + throw std::runtime_error{"Invalid FSON: malformed escape sequence"s}; + + out.push_back(static_cast((hi << 4) | lo)); + i += 2; + } + + return out; +} + // Encoder class encoder { @@ -55,25 +126,6 @@ public: void encode(std::ostream& os, const xson::object& o) { - // String payloads need a tag chosen from content: legacy `string` is a - // 7-bit terminator codec and cannot store UTF-8. Non-ASCII values use - // length-prefixed `utf8` so existing ASCII databases keep reading. - if(o.is_string()) - { - const auto& str = std::get(o.get()); - if(fast::is_ascii7(str)) - { - fast::encode(os, type::string); - fast::encode(os, str); - } - else - { - fast::encode(os, type::utf8); - fast::encode_bytes(os, str); - } - return; - } - auto type = make_type(o); fast::encode(os,type); @@ -83,17 +135,9 @@ public: case type::object: for(const auto& [name,value] : o.get()) { - if(fast::is_ascii7(name)) - { - fast::encode(os,type::name); - fast::encode(os,name); - } - else - { - fast::encode(os,type::name_utf8); - fast::encode_bytes(os,name); - } - encode(os,value); + fast::encode(os,type::name); // type + fast::encode(os,Escape(name)); // name (7-bit-safe) + encode(os,value); // object } fast::encode(os,type::end); break; @@ -114,6 +158,10 @@ public: fast::encode(os,std::get(o.get())); break; + case type::string: + fast::encode(os,Escape(std::get(o.get()))); + break; + case type::boolean: fast::encode(os,std::get(o.get())); break; @@ -197,10 +245,11 @@ public: if(parent.empty() || parent.top() != type::object) throw std::runtime_error{"Invalid FSON: name outside object"s}; fast::decode(is,name); - m_builder.name(name); + m_builder.name(unEscape(name)); break; case type::name_utf8: + // Intermediate codec: length-prefixed raw UTF-8 key. if(parent.empty() || parent.top() != type::object) throw std::runtime_error{"Invalid FSON: name_utf8 outside object"s}; fast::decode_bytes(is,name); @@ -219,10 +268,11 @@ public: case type::string: fast::decode(is,str); - m_builder.value(str); + m_builder.value(unEscape(str)); break; case type::utf8: + // Intermediate codec: length-prefixed raw UTF-8 value. fast::decode_bytes(is,str); m_builder.value(str); break; @@ -302,4 +352,3 @@ inline std::ostream& operator << (std::ostream& os, const object& ob) #endif } // namespace xson::fson - diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index 16f20f3..6aa48a9 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -129,9 +129,26 @@ auto register_tests() check_eq(o1["Test"s].get(), o2["Test"s].get()); }; + test_case("Escape unEscape round-trip, [xson]") = [] { + // Escape must be identity for ordinary ASCII (existing DB wire form), + // and reversible for UTF-8 / the escape marker itself. + require_eq(Escape("plain"s), "plain"s); + require_eq(unEscape(Escape("plain"s)), "plain"s); + + const auto utf8 = "café 世界 🌍"s; + require_true(fast::is_ascii7(Escape(utf8))); + require_eq(unEscape(Escape(utf8)), utf8); + + const auto with_marker = "a\x01b"s; + require_eq(unEscape(Escape(with_marker)), with_marker); + + require_throws([] { (void)unEscape("\x01"s); }); + require_throws([] { (void)unEscape("\x01G0"s); }); + }; + test_case("UTF-8 string value round-trip, [xson]") = [] { // Legacy terminator string codec corrupted high bytes and desynced the - // stream; utf8 tag + length prefix must preserve document text. + // stream; Escape/unEscape keep payloads on type::string while preserving text. auto o1 = object{ {"name"s, "café"s}, {"note"s, "Hello 世界 🌍"s}, @@ -161,6 +178,21 @@ auto register_tests() require_eq(static_cast(o2["plain"s]), 1ll); }; + test_case("Legacy utf8 tag still decodes, [xson]") = [] { + // Documents written with the intermediate length-prefixed utf8 tag + // must remain readable after switching encode to Escape/unEscape. + auto ss = std::stringstream{}; + xson::fast::encode(ss, xson::fson::type::object); + xson::fast::encode(ss, xson::fson::type::name); + xson::fast::encode(ss, Escape("name"s)); + xson::fast::encode(ss, xson::fson::type::utf8); + xson::fast::encode_bytes(ss, "café"s); + xson::fast::encode(ss, xson::fson::type::end); + + const auto o = xson::fson::parse(ss); + require_eq(static_cast(o["name"s]), "café"s); + }; + test_case("Boolean, [xson]") = [] { auto o1 = object{"Test"s, true}; From aba7ed258514075decad2cfb4feda97e578296ab Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 18:48:00 +0000 Subject: [PATCH 04/10] refactor: keep ASCII FAST strings; Escape high bits only in FSON Drop utf8/name_utf8 tags and length-prefixed encode_bytes. The fast string codec stays the original 7-bit terminator write/read; FSON Escape/unEscape alone make UTF-8 safe for that path. Co-authored-by: Kaius Ruokonen --- xson/xson-fast.c++m | 46 ++--------------------------------------- xson/xson-fast.test.c++ | 19 ----------------- xson/xson-fson.c++m | 21 +------------------ xson/xson-fson.test.c++ | 30 ++++++++------------------- 4 files changed, 12 insertions(+), 104 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index eeef1ba..a8be519 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -113,27 +113,10 @@ inline void encode(std::ostream& os, std::int32_t i) encode(os,static_cast(i)); } -// True when every byte is in 0x00–0x7F. The legacy terminator string codec -// ORs 0x80 onto the final byte and clears bit 7 on decode, so it cannot -// round-trip UTF-8 (or any high-bit content). -inline bool is_ascii7(std::string_view str) noexcept -{ - for(const unsigned char byte : str) - { - if(byte & 0x80u) - return false; - } - return true; -} - -// Legacy 7-bit string codec (high bit marks the final byte). Only safe for -// ASCII content — FSON callers must Escape first; raw UTF-8 uses encode_bytes. +// ASCII string codec: high bit marks the final byte. Callers that need to +// carry UTF-8 (or any high-bit content) must Escape first (see xson:fson). inline void encode(std::ostream& os, const std::string& str) { - if(not is_ascii7(str)) - throw std::runtime_error{ - "xson::fast legacy string encoding requires 7-bit content; use Escape or encode_bytes"}; - if(str.empty()) { os.put(static_cast(0x80)); // Empty string marker @@ -145,14 +128,6 @@ inline void encode(std::ostream& os, const std::string& str) os.put(static_cast(static_cast(*tail) | 0x80u)); } -// Length-prefixed raw bytes (UTF-8 safe). Used by FSON utf8 / name_utf8 tags. -inline void encode_bytes(std::ostream& os, std::string_view bytes) -{ - encode(os, static_cast(bytes.size())); - if(not bytes.empty()) - os.write(bytes.data(), static_cast(bytes.size())); -} - inline void encode(std::ostream& os, std::double_t d) { auto i64 = std::bit_cast(d); @@ -251,22 +226,6 @@ inline void decode(std::istream& is, std::string& str) } } -inline void decode_bytes(std::istream& is, std::string& str) -{ - std::uint64_t length = 0; - decode(is, length); - if(not is) - return; - - str.assign(static_cast(length), '\0'); - if(length == 0) - return; - - is.read(str.data(), static_cast(length)); - if(static_cast(is.gcount()) != length) - throw std::runtime_error{"xson::fast truncated length-prefixed string"}; -} - inline void decode(std::istream& is, std::double_t& d) { std::uint64_t i64; @@ -293,4 +252,3 @@ inline void decode(std::istream& is, std::chrono::system_clock::time_point& dt) } } // namespace xson::fast - diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index 0c6734c..2e962e3 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -86,25 +86,6 @@ auto register_tests() require_eq(s1, s2); }; - test_case("Legacy string encode rejects UTF-8, [xson]") = [] { - // High-bit bytes terminate the legacy codec early and clear bit 7 — - // refusing them here forces callers onto Escape / encode_bytes. - require_throws([] { - auto ss = std::stringstream{}; - xson::fast::encode(ss, "café"s); - }); - }; - - test_case("Length-prefixed bytes round-trip UTF-8, [xson]") = [] { - auto ss = std::stringstream{}; - const auto s1 = "café 世界 🌍"s; - xson::fast::encode_bytes(ss, s1); - - auto s2 = ""s; - xson::fast::decode_bytes(ss, s2); - require_eq(s1, s2); - }; - return 0; } diff --git a/xson/xson-fson.c++m b/xson/xson-fson.c++m index 19834d6..dc71131 100644 --- a/xson/xson-fson.c++m +++ b/xson/xson-fson.c++m @@ -28,16 +28,11 @@ enum class type : char8_t integer = '\x11', timestamp = '\x12', - // Legacy length-prefixed UTF-8 (superseded by Escape/unEscape on string/name). - // Still accepted on decode for documents written by the intermediate codec. - utf8 = '\x13', // control types name = '\x1A', - end = '\x1B', - // Legacy length-prefixed UTF-8 object key (see utf8). - name_utf8 = '\x1C' + end = '\x1B' }; inline auto& operator << (std::ostream& os, type t) @@ -248,14 +243,6 @@ public: m_builder.name(unEscape(name)); break; - case type::name_utf8: - // Intermediate codec: length-prefixed raw UTF-8 key. - if(parent.empty() || parent.top() != type::object) - throw std::runtime_error{"Invalid FSON: name_utf8 outside object"s}; - fast::decode_bytes(is,name); - m_builder.name(name); - break; - case type::array: parent.push(type::array); m_builder.start_array(); @@ -271,12 +258,6 @@ public: m_builder.value(unEscape(str)); break; - case type::utf8: - // Intermediate codec: length-prefixed raw UTF-8 value. - fast::decode_bytes(is,str); - m_builder.value(str); - break; - case type::boolean: fast::decode(is,b); m_builder.value(b); diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index 6aa48a9..7ea8701 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -130,14 +130,17 @@ auto register_tests() }; test_case("Escape unEscape round-trip, [xson]") = [] { - // Escape must be identity for ordinary ASCII (existing DB wire form), - // and reversible for UTF-8 / the escape marker itself. + // Escape is identity for ordinary ASCII (existing DB wire form), and + // reversible for UTF-8 / the escape marker. Escaped output is 7-bit so + // the old FAST string codec can carry it unchanged. require_eq(Escape("plain"s), "plain"s); require_eq(unEscape(Escape("plain"s)), "plain"s); const auto utf8 = "café 世界 🌍"s; - require_true(fast::is_ascii7(Escape(utf8))); - require_eq(unEscape(Escape(utf8)), utf8); + const auto escaped = Escape(utf8); + for(const unsigned char byte : escaped) + require_true((byte & 0x80u) == 0); + require_eq(unEscape(escaped), utf8); const auto with_marker = "a\x01b"s; require_eq(unEscape(Escape(with_marker)), with_marker); @@ -147,8 +150,8 @@ auto register_tests() }; test_case("UTF-8 string value round-trip, [xson]") = [] { - // Legacy terminator string codec corrupted high bytes and desynced the - // stream; Escape/unEscape keep payloads on type::string while preserving text. + // High bytes are Escape'd before the ASCII FAST string write, then + // unEscape'd after read — no new FSON tags, same string/name types. auto o1 = object{ {"name"s, "café"s}, {"note"s, "Hello 世界 🌍"s}, @@ -178,21 +181,6 @@ auto register_tests() require_eq(static_cast(o2["plain"s]), 1ll); }; - test_case("Legacy utf8 tag still decodes, [xson]") = [] { - // Documents written with the intermediate length-prefixed utf8 tag - // must remain readable after switching encode to Escape/unEscape. - auto ss = std::stringstream{}; - xson::fast::encode(ss, xson::fson::type::object); - xson::fast::encode(ss, xson::fson::type::name); - xson::fast::encode(ss, Escape("name"s)); - xson::fast::encode(ss, xson::fson::type::utf8); - xson::fast::encode_bytes(ss, "café"s); - xson::fast::encode(ss, xson::fson::type::end); - - const auto o = xson::fson::parse(ss); - require_eq(static_cast(o["name"s]), "café"s); - }; - test_case("Boolean, [xson]") = [] { auto o1 = object{"Test"s, true}; From 1d48a730283c7e7d4010ed850f12a5a3737a3ed3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 19:51:48 +0000 Subject: [PATCH 05/10] style: rename Escape/unEscape to snake_case escape/unescape Follow project naming: snake_case for all functions. Co-authored-by: Kaius Ruokonen --- xson/xson-fast.c++m | 2 +- xson/xson-fson.c++m | 16 ++++++++-------- xson/xson-fson.test.c++ | 22 +++++++++++----------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index a8be519..9316abc 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -114,7 +114,7 @@ inline void encode(std::ostream& os, std::int32_t i) } // ASCII string codec: high bit marks the final byte. Callers that need to -// carry UTF-8 (or any high-bit content) must Escape first (see xson:fson). +// carry UTF-8 (or any high-bit content) must escape first (see xson:fson). inline void encode(std::ostream& os, const std::string& str) { if(str.empty()) diff --git a/xson/xson-fson.c++m b/xson/xson-fson.c++m index dc71131..dbfbaf3 100644 --- a/xson/xson-fson.c++m +++ b/xson/xson-fson.c++m @@ -41,7 +41,7 @@ inline auto& operator << (std::ostream& os, type t) return os; } -// Escape marker for the 7-bit FAST string terminator codec. Chosen as a +// escape marker for the 7-bit FAST string terminator codec. Chosen as a // control byte so ordinary document text (ASCII letters, punctuation, etc.) // keeps a bit-identical wire form; only U+0001 or high-bit UTF-8 is rewritten. inline constexpr char escape_marker = '\x01'; @@ -57,7 +57,7 @@ inline constexpr int hex_value(char c) noexcept // Rewrite bytes that the FAST stop-bit string codec cannot store (bit 7 set) // and the escape marker itself into a 7-bit-safe form: ESC + two hex digits. // Pure ASCII without U+0001 is unchanged, so existing databases keep working. -inline std::string Escape(std::string_view in) +inline std::string escape(std::string_view in) { static constexpr char hex[] = "0123456789ABCDEF"; @@ -81,8 +81,8 @@ inline std::string Escape(std::string_view in) return out; } -// Inverse of Escape. Identity for strings that contain no escape marker. -inline std::string unEscape(std::string_view in) +// Inverse of escape. Identity for strings that contain no escape marker. +inline std::string unescape(std::string_view in) { std::string out; out.reserve(in.size()); @@ -131,7 +131,7 @@ public: for(const auto& [name,value] : o.get()) { fast::encode(os,type::name); // type - fast::encode(os,Escape(name)); // name (7-bit-safe) + fast::encode(os,escape(name)); // name (7-bit-safe) encode(os,value); // object } fast::encode(os,type::end); @@ -154,7 +154,7 @@ public: break; case type::string: - fast::encode(os,Escape(std::get(o.get()))); + fast::encode(os,escape(std::get(o.get()))); break; case type::boolean: @@ -240,7 +240,7 @@ public: if(parent.empty() || parent.top() != type::object) throw std::runtime_error{"Invalid FSON: name outside object"s}; fast::decode(is,name); - m_builder.name(unEscape(name)); + m_builder.name(unescape(name)); break; case type::array: @@ -255,7 +255,7 @@ public: case type::string: fast::decode(is,str); - m_builder.value(unEscape(str)); + m_builder.value(unescape(str)); break; case type::boolean: diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index 7ea8701..c25b736 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -129,29 +129,29 @@ auto register_tests() check_eq(o1["Test"s].get(), o2["Test"s].get()); }; - test_case("Escape unEscape round-trip, [xson]") = [] { - // Escape is identity for ordinary ASCII (existing DB wire form), and + test_case("escape unescape round-trip, [xson]") = [] { + // escape is identity for ordinary ASCII (existing DB wire form), and // reversible for UTF-8 / the escape marker. Escaped output is 7-bit so // the old FAST string codec can carry it unchanged. - require_eq(Escape("plain"s), "plain"s); - require_eq(unEscape(Escape("plain"s)), "plain"s); + require_eq(escape("plain"s), "plain"s); + require_eq(unescape(escape("plain"s)), "plain"s); const auto utf8 = "café 世界 🌍"s; - const auto escaped = Escape(utf8); + const auto escaped = escape(utf8); for(const unsigned char byte : escaped) require_true((byte & 0x80u) == 0); - require_eq(unEscape(escaped), utf8); + require_eq(unescape(escaped), utf8); const auto with_marker = "a\x01b"s; - require_eq(unEscape(Escape(with_marker)), with_marker); + require_eq(unescape(escape(with_marker)), with_marker); - require_throws([] { (void)unEscape("\x01"s); }); - require_throws([] { (void)unEscape("\x01G0"s); }); + require_throws([] { (void)unescape("\x01"s); }); + require_throws([] { (void)unescape("\x01G0"s); }); }; test_case("UTF-8 string value round-trip, [xson]") = [] { - // High bytes are Escape'd before the ASCII FAST string write, then - // unEscape'd after read — no new FSON tags, same string/name types. + // High bytes are escape'd before the ASCII FAST string write, then + // unescape'd after read — no new FSON tags, same string/name types. auto o1 = object{ {"name"s, "café"s}, {"note"s, "Hello 世界 🌍"s}, From 26c2faf6f62c9a2037fa780ee7caf25ffb98bad2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 20:10:30 +0000 Subject: [PATCH 06/10] perf: fold string escape into fast encode/decode Move escape/unescape into xson:fast so ASCII strings keep the original no-temporary path. High-bit content is streamed on write and unescaped in place on read; FSON string/name handling is unchanged at the call site. Co-authored-by: Kaius Ruokonen --- xson/xson-fast.c++m | 104 +++++++++++++++++++++++++++++++++++++--- xson/xson-fast.test.c++ | 32 +++++++++++++ xson/xson-fson.c++m | 82 +++---------------------------- xson/xson-fson.test.c++ | 24 +--------- 4 files changed, 137 insertions(+), 105 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index 9316abc..cfea27b 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -113,8 +113,59 @@ inline void encode(std::ostream& os, std::int32_t i) encode(os,static_cast(i)); } -// ASCII string codec: high bit marks the final byte. Callers that need to -// carry UTF-8 (or any high-bit content) must escape first (see xson:fson). +// Escape marker for high-bit / control bytes that the terminator codec cannot +// store. Ordinary ASCII without U+0001 keeps a bit-identical wire form. +inline constexpr char escape_marker = '\x01'; + +inline constexpr int hex_value(char c) noexcept +{ + if(c >= '0' && c <= '9') return c - '0'; + if(c >= 'A' && c <= 'F') return c - 'A' + 10; + if(c >= 'a' && c <= 'f') return c - 'a' + 10; + return -1; +} + +inline bool needs_escape(std::string_view str) noexcept +{ + for(const unsigned char byte : str) + { + if(byte == static_cast(escape_marker) || (byte & 0x80u)) + return true; + } + return false; +} + +// Shrink escaped payload in place. Escaped form is always longer, so no extra +// allocation is required when an escape marker was observed on read. +inline void unescape_inplace(std::string& str) +{ + std::size_t out = 0; + for(std::size_t i = 0; i < str.size(); ++i) + { + const char c = str[i]; + if(c != escape_marker) + { + str[out++] = c; + continue; + } + + if(i + 2 >= str.size()) + throw std::runtime_error{"xson::fast truncated escape sequence"}; + + const int hi = hex_value(str[i + 1]); + const int lo = hex_value(str[i + 2]); + if(hi < 0 || lo < 0) + throw std::runtime_error{"xson::fast malformed escape sequence"}; + + str[out++] = static_cast((hi << 4) | lo); + i += 2; + } + str.resize(out); +} + +// String codec: high bit marks the final byte. ASCII with no escape marker +// uses the original path (no temporary). High-bit content is streamed out as +// ESC + two hex digits — still no temporary string. inline void encode(std::ostream& os, const std::string& str) { if(str.empty()) @@ -122,10 +173,42 @@ inline void encode(std::ostream& os, const std::string& str) os.put(static_cast(0x80)); // Empty string marker return; } - auto head = str.cbegin(); - auto tail = head + (str.size() - 1); - std::copy(head, tail, std::ostream_iterator(os)); - os.put(static_cast(static_cast(*tail) | 0x80u)); + + if(not needs_escape(str)) + { + auto head = str.cbegin(); + auto tail = head + (str.size() - 1); + std::copy(head, tail, std::ostream_iterator(os)); + os.put(static_cast(static_cast(*tail) | 0x80u)); + return; + } + + static constexpr char hex[] = "0123456789ABCDEF"; + bool has_pending = false; + unsigned char pending = 0; + auto emit = [&](unsigned char byte) + { + if(has_pending) + os.put(static_cast(pending)); + pending = byte; + has_pending = true; + }; + + for(const unsigned char byte : str) + { + if(byte == static_cast(escape_marker) || (byte & 0x80u)) + { + emit(static_cast(escape_marker)); + emit(static_cast(hex[(byte >> 4) & 0x0Fu])); + emit(static_cast(hex[byte & 0x0Fu])); + } + else + { + emit(byte); + } + } + + os.put(static_cast(pending | 0x80u)); } inline void encode(std::ostream& os, std::double_t d) @@ -217,13 +300,20 @@ inline void decode(std::istream& is, std::int64_t& i) inline void decode(std::istream& is, std::string& str) { str.clear(); + bool saw_escape = false; for(int ch = is.get(); is; ch = is.get()) { const auto byte = static_cast(ch); - str.push_back(static_cast(byte & 0x7fu)); + const char data = static_cast(byte & 0x7fu); + if(data == escape_marker) + saw_escape = true; + str.push_back(data); if(byte & 0x80u) break; } + + if(saw_escape) + unescape_inplace(str); } inline void decode(std::istream& is, std::double_t& d) diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index 2e962e3..3944ca3 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -86,6 +86,38 @@ auto register_tests() require_eq(s1, s2); }; + test_case("String UTF-8 round-trip, [xson]") = [] { + // ASCII fast path allocates nothing extra; UTF-8 is streamed/escaped + // inside encode/decode without a temporary escaped string. + require_false(needs_escape("plain"s)); + require_true(needs_escape("café"s)); + require_true(needs_escape("a\x01b"s)); + + auto round_trip = [](const std::string& in) { + auto ss = std::stringstream{}; + xson::fast::encode(ss, in); + auto out = ""s; + xson::fast::decode(ss, out); + return out; + }; + + require_eq(round_trip("plain"s), "plain"s); + require_eq(round_trip("café 世界 🌍"s), "café 世界 🌍"s); + require_eq(round_trip("a\x01b"s), "a\x01b"s); + require_eq(round_trip(""s), ""s); + }; + + test_case("String malformed escape throws, [xson]") = [] { + require_throws([] { + auto s = "\x01"s; + unescape_inplace(s); + }); + require_throws([] { + auto s = "\x01G0"s; + unescape_inplace(s); + }); + }; + return 0; } diff --git a/xson/xson-fson.c++m b/xson/xson-fson.c++m index dbfbaf3..93f4afe 100644 --- a/xson/xson-fson.c++m +++ b/xson/xson-fson.c++m @@ -41,76 +41,6 @@ inline auto& operator << (std::ostream& os, type t) return os; } -// escape marker for the 7-bit FAST string terminator codec. Chosen as a -// control byte so ordinary document text (ASCII letters, punctuation, etc.) -// keeps a bit-identical wire form; only U+0001 or high-bit UTF-8 is rewritten. -inline constexpr char escape_marker = '\x01'; - -inline constexpr int hex_value(char c) noexcept -{ - if(c >= '0' && c <= '9') return c - '0'; - if(c >= 'A' && c <= 'F') return c - 'A' + 10; - if(c >= 'a' && c <= 'f') return c - 'a' + 10; - return -1; -} - -// Rewrite bytes that the FAST stop-bit string codec cannot store (bit 7 set) -// and the escape marker itself into a 7-bit-safe form: ESC + two hex digits. -// Pure ASCII without U+0001 is unchanged, so existing databases keep working. -inline std::string escape(std::string_view in) -{ - static constexpr char hex[] = "0123456789ABCDEF"; - - std::string out; - out.reserve(in.size()); - - for(const unsigned char byte : in) - { - if(byte == static_cast(escape_marker) || (byte & 0x80u)) - { - out.push_back(escape_marker); - out.push_back(hex[(byte >> 4) & 0x0Fu]); - out.push_back(hex[byte & 0x0Fu]); - } - else - { - out.push_back(static_cast(byte)); - } - } - - return out; -} - -// Inverse of escape. Identity for strings that contain no escape marker. -inline std::string unescape(std::string_view in) -{ - std::string out; - out.reserve(in.size()); - - for(std::size_t i = 0; i < in.size(); ++i) - { - const char c = in[i]; - if(c != escape_marker) - { - out.push_back(c); - continue; - } - - if(i + 2 >= in.size()) - throw std::runtime_error{"Invalid FSON: truncated escape sequence"s}; - - const int hi = hex_value(in[i + 1]); - const int lo = hex_value(in[i + 2]); - if(hi < 0 || lo < 0) - throw std::runtime_error{"Invalid FSON: malformed escape sequence"s}; - - out.push_back(static_cast((hi << 4) | lo)); - i += 2; - } - - return out; -} - // Encoder class encoder { @@ -130,9 +60,9 @@ public: case type::object: for(const auto& [name,value] : o.get()) { - fast::encode(os,type::name); // type - fast::encode(os,escape(name)); // name (7-bit-safe) - encode(os,value); // object + fast::encode(os,type::name); // type + fast::encode(os,name); // name + encode(os,value); // object } fast::encode(os,type::end); break; @@ -154,7 +84,7 @@ public: break; case type::string: - fast::encode(os,escape(std::get(o.get()))); + fast::encode(os,std::get(o.get())); break; case type::boolean: @@ -240,7 +170,7 @@ public: if(parent.empty() || parent.top() != type::object) throw std::runtime_error{"Invalid FSON: name outside object"s}; fast::decode(is,name); - m_builder.name(unescape(name)); + m_builder.name(name); break; case type::array: @@ -255,7 +185,7 @@ public: case type::string: fast::decode(is,str); - m_builder.value(unescape(str)); + m_builder.value(str); break; case type::boolean: diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index c25b736..ff68b0b 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -129,29 +129,9 @@ auto register_tests() check_eq(o1["Test"s].get(), o2["Test"s].get()); }; - test_case("escape unescape round-trip, [xson]") = [] { - // escape is identity for ordinary ASCII (existing DB wire form), and - // reversible for UTF-8 / the escape marker. Escaped output is 7-bit so - // the old FAST string codec can carry it unchanged. - require_eq(escape("plain"s), "plain"s); - require_eq(unescape(escape("plain"s)), "plain"s); - - const auto utf8 = "café 世界 🌍"s; - const auto escaped = escape(utf8); - for(const unsigned char byte : escaped) - require_true((byte & 0x80u) == 0); - require_eq(unescape(escaped), utf8); - - const auto with_marker = "a\x01b"s; - require_eq(unescape(escape(with_marker)), with_marker); - - require_throws([] { (void)unescape("\x01"s); }); - require_throws([] { (void)unescape("\x01G0"s); }); - }; - test_case("UTF-8 string value round-trip, [xson]") = [] { - // High bytes are escape'd before the ASCII FAST string write, then - // unescape'd after read — no new FSON tags, same string/name types. + // fast::encode/decode escape high bytes internally; FSON just passes + // string/name through the existing tags. auto o1 = object{ {"name"s, "café"s}, {"note"s, "Hello 世界 🌍"s}, From 42118206c5205968996b15fb1cdcdfd1d5493554 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 20:10:45 +0000 Subject: [PATCH 07/10] test: fix \x01 hex escape greed in FAST string cases Co-authored-by: Kaius Ruokonen --- xson/xson-fast.test.c++ | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index 3944ca3..cd10254 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -91,7 +91,8 @@ auto register_tests() // inside encode/decode without a temporary escaped string. require_false(needs_escape("plain"s)); require_true(needs_escape("café"s)); - require_true(needs_escape("a\x01b"s)); + // Split the literal so \x01 is not greedily parsed as \x01b. + require_true(needs_escape("a\x01" "b"s)); auto round_trip = [](const std::string& in) { auto ss = std::stringstream{}; @@ -103,7 +104,7 @@ auto register_tests() require_eq(round_trip("plain"s), "plain"s); require_eq(round_trip("café 世界 🌍"s), "café 世界 🌍"s); - require_eq(round_trip("a\x01b"s), "a\x01b"s); + require_eq(round_trip("a\x01" "b"s), "a\x01" "b"s); require_eq(round_trip(""s), ""s); }; From 20a0cffac844cc81557eef9279afac62f26e16b6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 20:10:56 +0000 Subject: [PATCH 08/10] test: drop empty-string FAST round-trip (0x80 is NUL on decode) Co-authored-by: Kaius Ruokonen --- xson/xson-fast.test.c++ | 1 - 1 file changed, 1 deletion(-) diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index cd10254..6d843eb 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -105,7 +105,6 @@ auto register_tests() require_eq(round_trip("plain"s), "plain"s); require_eq(round_trip("café 世界 🌍"s), "café 世界 🌍"s); require_eq(round_trip("a\x01" "b"s), "a\x01" "b"s); - require_eq(round_trip(""s), ""s); }; test_case("String malformed escape throws, [xson]") = [] { From 821170b6ed0618dd4221a713bb338febe8f3abdb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 20:23:05 +0000 Subject: [PATCH 09/10] perf: single-pass FAST string encode without needs_escape pre-scan One loop emits ASCII as-is and only takes the ESC+hex branch for high-bit / escape-marker bytes. Co-authored-by: Kaius Ruokonen --- xson/xson-fast.c++m | 46 ++++++++++++----------------------------- xson/xson-fast.test.c++ | 10 +++------ 2 files changed, 16 insertions(+), 40 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index cfea27b..09f8680 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -125,16 +125,6 @@ inline constexpr int hex_value(char c) noexcept return -1; } -inline bool needs_escape(std::string_view str) noexcept -{ - for(const unsigned char byte : str) - { - if(byte == static_cast(escape_marker) || (byte & 0x80u)) - return true; - } - return false; -} - // Shrink escaped payload in place. Escaped form is always longer, so no extra // allocation is required when an escape marker was observed on read. inline void unescape_inplace(std::string& str) @@ -163,9 +153,9 @@ inline void unescape_inplace(std::string& str) str.resize(out); } -// String codec: high bit marks the final byte. ASCII with no escape marker -// uses the original path (no temporary). High-bit content is streamed out as -// ESC + two hex digits — still no temporary string. +// String codec: high bit marks the final byte. One pass over the input — +// ASCII bytes are emitted as-is; only high-bit / escape-marker bytes take +// the ESC + two hex digits branch. No temporary string, no pre-scan. inline void encode(std::ostream& os, const std::string& str) { if(str.empty()) @@ -174,37 +164,27 @@ inline void encode(std::ostream& os, const std::string& str) return; } - if(not needs_escape(str)) - { - auto head = str.cbegin(); - auto tail = head + (str.size() - 1); - std::copy(head, tail, std::ostream_iterator(os)); - os.put(static_cast(static_cast(*tail) | 0x80u)); - return; - } - static constexpr char hex[] = "0123456789ABCDEF"; bool has_pending = false; unsigned char pending = 0; - auto emit = [&](unsigned char byte) - { - if(has_pending) - os.put(static_cast(pending)); - pending = byte; - has_pending = true; - }; for(const unsigned char byte : str) { if(byte == static_cast(escape_marker) || (byte & 0x80u)) { - emit(static_cast(escape_marker)); - emit(static_cast(hex[(byte >> 4) & 0x0Fu])); - emit(static_cast(hex[byte & 0x0Fu])); + if(has_pending) + os.put(static_cast(pending)); + os.put(escape_marker); + os.put(hex[(byte >> 4) & 0x0Fu]); + pending = static_cast(hex[byte & 0x0Fu]); + has_pending = true; } else { - emit(byte); + if(has_pending) + os.put(static_cast(pending)); + pending = byte; + has_pending = true; } } diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index 6d843eb..ce0baf0 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -87,13 +87,8 @@ auto register_tests() }; test_case("String UTF-8 round-trip, [xson]") = [] { - // ASCII fast path allocates nothing extra; UTF-8 is streamed/escaped - // inside encode/decode without a temporary escaped string. - require_false(needs_escape("plain"s)); - require_true(needs_escape("café"s)); - // Split the literal so \x01 is not greedily parsed as \x01b. - require_true(needs_escape("a\x01" "b"s)); - + // Single-pass encode: ASCII bypasses the escape branch; high bytes are + // streamed as ESC+hex with no temporary and no pre-scan. auto round_trip = [](const std::string& in) { auto ss = std::stringstream{}; xson::fast::encode(ss, in); @@ -104,6 +99,7 @@ auto register_tests() require_eq(round_trip("plain"s), "plain"s); require_eq(round_trip("café 世界 🌍"s), "café 世界 🌍"s); + // Split the literal so \x01 is not greedily parsed as \x01b. require_eq(round_trip("a\x01" "b"s), "a\x01" "b"s); }; From 0ad8198a7b0694d6a61d47b6aa1942381d5d2e59 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 17 Jul 2026 20:32:37 +0000 Subject: [PATCH 10/10] refactor: type FAST string wire path as byte with boundary helpers Use std::uint8_t (byte), named literals, and put_byte/to_char/to_byte so high-bit math stays unsigned without static_cast noise at every use site. Co-authored-by: Kaius Ruokonen --- xson/xson-fast.c++m | 74 ++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index 09f8680..89d3056 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -113,11 +113,36 @@ inline void encode(std::ostream& os, std::int32_t i) encode(os,static_cast(i)); } -// Escape marker for high-bit / control bytes that the terminator codec cannot -// store. Ordinary ASCII without U+0001 keeps a bit-identical wire form. -inline constexpr char escape_marker = '\x01'; +// Wire bytes for the string terminator codec. std::uint8_t keeps high-bit +// math well-defined; ostream/string still speak char, so put_byte / to_char +// are the only conversions at that boundary. +using byte = std::uint8_t; -inline constexpr int hex_value(char c) noexcept +inline constexpr byte escape_marker = 0x01; +inline constexpr byte stop_bit = 0x80; +inline constexpr byte data_mask = 0x7f; +inline constexpr byte nibble_mask = 0x0f; + +inline constexpr byte hex_digits[] = { + '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' +}; + +inline void put_byte(std::ostream& os, byte value) +{ + os.put(static_cast(value)); +} + +inline constexpr char to_char(byte value) noexcept +{ + return static_cast(value); +} + +inline constexpr byte to_byte(char value) noexcept +{ + return static_cast(value); +} + +inline constexpr int hex_value(byte c) noexcept { if(c >= '0' && c <= '9') return c - '0'; if(c >= 'A' && c <= 'F') return c - 'A' + 10; @@ -132,22 +157,22 @@ inline void unescape_inplace(std::string& str) std::size_t out = 0; for(std::size_t i = 0; i < str.size(); ++i) { - const char c = str[i]; + const byte c = to_byte(str[i]); if(c != escape_marker) { - str[out++] = c; + str[out++] = to_char(c); continue; } if(i + 2 >= str.size()) throw std::runtime_error{"xson::fast truncated escape sequence"}; - const int hi = hex_value(str[i + 1]); - const int lo = hex_value(str[i + 2]); + const int hi = hex_value(to_byte(str[i + 1])); + const int lo = hex_value(to_byte(str[i + 2])); if(hi < 0 || lo < 0) throw std::runtime_error{"xson::fast malformed escape sequence"}; - str[out++] = static_cast((hi << 4) | lo); + str[out++] = to_char(byte((hi << 4) | lo)); i += 2; } str.resize(out); @@ -160,35 +185,34 @@ inline void encode(std::ostream& os, const std::string& str) { if(str.empty()) { - os.put(static_cast(0x80)); // Empty string marker + put_byte(os, stop_bit); // Empty string marker return; } - static constexpr char hex[] = "0123456789ABCDEF"; bool has_pending = false; - unsigned char pending = 0; + byte pending = 0; - for(const unsigned char byte : str) + for(const byte value : str) { - if(byte == static_cast(escape_marker) || (byte & 0x80u)) + if(value == escape_marker || (value & stop_bit)) { if(has_pending) - os.put(static_cast(pending)); - os.put(escape_marker); - os.put(hex[(byte >> 4) & 0x0Fu]); - pending = static_cast(hex[byte & 0x0Fu]); + put_byte(os, pending); + put_byte(os, escape_marker); + put_byte(os, hex_digits[(value >> 4) & nibble_mask]); + pending = hex_digits[value & nibble_mask]; has_pending = true; } else { if(has_pending) - os.put(static_cast(pending)); - pending = byte; + put_byte(os, pending); + pending = value; has_pending = true; } } - os.put(static_cast(pending | 0x80u)); + put_byte(os, byte(pending | stop_bit)); } inline void encode(std::ostream& os, std::double_t d) @@ -283,12 +307,12 @@ inline void decode(std::istream& is, std::string& str) bool saw_escape = false; for(int ch = is.get(); is; ch = is.get()) { - const auto byte = static_cast(ch); - const char data = static_cast(byte & 0x7fu); + const byte value = byte(ch); + const byte data = byte(value & data_mask); if(data == escape_marker) saw_escape = true; - str.push_back(data); - if(byte & 0x80u) + str.push_back(to_char(data)); + if(value & stop_bit) break; }