diff --git a/xson/xson-fast.c++m b/xson/xson-fast.c++m index e778a17..89d3056 100644 --- a/xson/xson-fast.c++m +++ b/xson/xson-fast.c++m @@ -113,18 +113,106 @@ inline void encode(std::ostream& os, std::int32_t i) encode(os,static_cast(i)); } +// Wire bytes for the string terminator codec. std::uint8_t keeps high-bit +// math well-defined; ostream/string still speak char, so put_byte / to_char +// are the only conversions at that boundary. +using byte = std::uint8_t; + +inline constexpr byte escape_marker = 0x01; +inline constexpr byte stop_bit = 0x80; +inline constexpr byte data_mask = 0x7f; +inline constexpr byte nibble_mask = 0x0f; + +inline constexpr byte hex_digits[] = { + '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' +}; + +inline void put_byte(std::ostream& os, byte value) +{ + os.put(static_cast(value)); +} + +inline constexpr char to_char(byte value) noexcept +{ + return static_cast(value); +} + +inline constexpr byte to_byte(char value) noexcept +{ + return static_cast(value); +} + +inline constexpr int hex_value(byte c) noexcept +{ + if(c >= '0' && c <= '9') return c - '0'; + if(c >= 'A' && c <= 'F') return c - 'A' + 10; + if(c >= 'a' && c <= 'f') return c - 'a' + 10; + return -1; +} + +// Shrink escaped payload in place. Escaped form is always longer, so no extra +// allocation is required when an escape marker was observed on read. +inline void unescape_inplace(std::string& str) +{ + std::size_t out = 0; + for(std::size_t i = 0; i < str.size(); ++i) + { + const byte c = to_byte(str[i]); + if(c != escape_marker) + { + str[out++] = to_char(c); + continue; + } + + if(i + 2 >= str.size()) + throw std::runtime_error{"xson::fast truncated escape sequence"}; + + const int hi = hex_value(to_byte(str[i + 1])); + const int lo = hex_value(to_byte(str[i + 2])); + if(hi < 0 || lo < 0) + throw std::runtime_error{"xson::fast malformed escape sequence"}; + + str[out++] = to_char(byte((hi << 4) | lo)); + i += 2; + } + str.resize(out); +} + +// String codec: high bit marks the final byte. One pass over the input — +// ASCII bytes are emitted as-is; only high-bit / escape-marker bytes take +// the ESC + two hex digits branch. No temporary string, no pre-scan. inline void encode(std::ostream& os, const std::string& str) { - if(str.empty()) { - os.put(0x80); // Empty string marker + put_byte(os, stop_bit); // Empty string marker return; } - auto head = str.cbegin(); - auto tail = head + (str.size() - 1); - std::copy(head, tail, std::ostream_iterator(os)); - os.put(*tail | 0x80); + + bool has_pending = false; + byte pending = 0; + + for(const byte value : str) + { + if(value == escape_marker || (value & stop_bit)) + { + if(has_pending) + put_byte(os, pending); + put_byte(os, escape_marker); + put_byte(os, hex_digits[(value >> 4) & nibble_mask]); + pending = hex_digits[value & nibble_mask]; + has_pending = true; + } + else + { + if(has_pending) + put_byte(os, pending); + pending = value; + has_pending = true; + } + } + + put_byte(os, byte(pending | stop_bit)); } inline void encode(std::ostream& os, std::double_t d) @@ -216,13 +304,20 @@ inline void decode(std::istream& is, std::int64_t& i) inline void decode(std::istream& is, std::string& str) { str.clear(); - for(char byte = is.get(); is; byte = is.get()) + bool saw_escape = false; + for(int ch = is.get(); is; ch = is.get()) { - str.push_back(byte & 0x7f); - if(byte & 0x80) + const byte value = byte(ch); + const byte data = byte(value & data_mask); + if(data == escape_marker) + saw_escape = true; + str.push_back(to_char(data)); + if(value & stop_bit) break; } - + + if(saw_escape) + unescape_inplace(str); } inline void decode(std::istream& is, std::double_t& d) @@ -251,4 +346,3 @@ inline void decode(std::istream& is, std::chrono::system_clock::time_point& dt) } } // namespace xson::fast - diff --git a/xson/xson-fast.test.c++ b/xson/xson-fast.test.c++ index 2e962e3..ce0baf0 100644 --- a/xson/xson-fast.test.c++ +++ b/xson/xson-fast.test.c++ @@ -86,6 +86,34 @@ auto register_tests() require_eq(s1, s2); }; + test_case("String UTF-8 round-trip, [xson]") = [] { + // Single-pass encode: ASCII bypasses the escape branch; high bytes are + // streamed as ESC+hex with no temporary and no pre-scan. + auto round_trip = [](const std::string& in) { + auto ss = std::stringstream{}; + xson::fast::encode(ss, in); + auto out = ""s; + xson::fast::decode(ss, out); + return out; + }; + + require_eq(round_trip("plain"s), "plain"s); + require_eq(round_trip("café 世界 🌍"s), "café 世界 🌍"s); + // Split the literal so \x01 is not greedily parsed as \x01b. + require_eq(round_trip("a\x01" "b"s), "a\x01" "b"s); + }; + + test_case("String malformed escape throws, [xson]") = [] { + require_throws([] { + auto s = "\x01"s; + unescape_inplace(s); + }); + require_throws([] { + auto s = "\x01G0"s; + unescape_inplace(s); + }); + }; + return 0; } diff --git a/xson/xson-fson.c++m b/xson/xson-fson.c++m index 20a00e1..93f4afe 100644 --- a/xson/xson-fson.c++m +++ b/xson/xson-fson.c++m @@ -263,4 +263,3 @@ inline std::ostream& operator << (std::ostream& os, const object& ob) #endif } // namespace xson::fson - diff --git a/xson/xson-fson.test.c++ b/xson/xson-fson.test.c++ index 354e21d..ff68b0b 100644 --- a/xson/xson-fson.test.c++ +++ b/xson/xson-fson.test.c++ @@ -129,6 +129,38 @@ auto register_tests() check_eq(o1["Test"s].get(), o2["Test"s].get()); }; + test_case("UTF-8 string value round-trip, [xson]") = [] { + // fast::encode/decode escape high bytes internally; FSON just passes + // string/name through the existing tags. + auto o1 = object{ + {"name"s, "café"s}, + {"note"s, "Hello 世界 🌍"s}, + {"ascii"s, "plain"s} + }; + + auto ss = std::stringstream{}; + xson::fson::encoder{}.encode(ss, o1); + const auto o2 = xson::fson::parse(ss); + + require_eq(static_cast(o2["name"s]), "café"s); + require_eq(static_cast(o2["note"s]), "Hello 世界 🌍"s); + require_eq(static_cast(o2["ascii"s]), "plain"s); + }; + + test_case("UTF-8 object key round-trip, [xson]") = [] { + auto o1 = object{}; + o1["名"s] = "value"s; + o1["plain"s] = 1ll; + + auto ss = std::stringstream{}; + xson::fson::encoder{}.encode(ss, o1); + const auto o2 = xson::fson::parse(ss); + + require_true(o2.has("名"s)); + require_eq(static_cast(o2["名"s]), "value"s); + require_eq(static_cast(o2["plain"s]), 1ll); + }; + test_case("Boolean, [xson]") = [] { auto o1 = object{"Test"s, true};