From 767d4ed92526db033b5a00d5a29f22cc8c1e76eb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 19 Aug 2025 16:52:11 -0600 Subject: [PATCH 1/3] Add Unicode replacement and surrogate tolerance flags Add YYJSON_READ_REPLACE_INVALID_UNICODE and YYJSON_READ_ALLOW_INVALID_SURROGATE flags to enable graceful handling of malformed Unicode in JSON strings. YYJSON_READ_REPLACE_INVALID_UNICODE replaces all invalid Unicode sequences with U+FFFD replacement character, ensuring output is always valid UTF-8. Handles invalid \uXXXX escapes, unpaired surrogates, invalid UTF-8 bytes, and control characters. YYJSON_READ_ALLOW_INVALID_SURROGATE permits unpaired surrogate code units in \uXXXX escapes, encoding them as UTF-8. Key changes: - Modified read_uni_esc() to accept flags parameter - Added replacement logic throughout string parsing paths - Replacement flag implicitly enables unicode tolerance flags - Added comprehensive test coverage - Flags not supported in incremental parsing mode This enables robust parsing of real-world JSON with encoding errors while maintaining strict standards compliance by default. Signed-off-by: Eduardo Silva --- src/yyjson.c | 156 +++++++++++++++++++++++++++++++++++++++++---- src/yyjson.h | 31 ++++++--- test/test_string.c | 34 ++++++++++ 3 files changed, 199 insertions(+), 22 deletions(-) diff --git a/src/yyjson.c b/src/yyjson.c index c16d925..25a5674 100644 --- a/src/yyjson.c +++ b/src/yyjson.c @@ -4657,7 +4657,8 @@ static_inline bool read_num(u8 **ptr, u8 **pre, yyjson_read_flag flg, *============================================================================*/ /** Read unicode escape sequence. */ -static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, const char **msg) { +static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, + const char **msg, yyjson_read_flag flg) { #define return_err(_end, _msg) *msg = _msg; *src_ptr = _end; return false u8 *src = *src_ptr; @@ -4667,6 +4668,15 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, const char **msg) { src += 2; /* skip `\u` */ if (unlikely(!hex_load_4(src, &hi))) { + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + usize cnt = 0; + while (cnt < 4 && char_is_hex(src[cnt])) cnt++; + src += cnt; + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + *src_ptr = src; + *dst_ptr = dst; + return true; + } return_err(src - 2, "invalid escaped sequence in string"); } src += 4; /* skip hex */ @@ -4682,18 +4692,83 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, const char **msg) { } else { *dst++ = (u8)hi; } - } else { + } else if ((hi & 0xFC00) == 0xD800) { /* a non-BMP character, represented as a surrogate pair */ - if (unlikely((hi & 0xFC00) != 0xD800)) { - return_err(src - 6, "invalid high surrogate in string"); - } if (unlikely(!byte_match_2(src, "\\u"))) { + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + *src_ptr = src; + *dst_ptr = dst; + return true; + } + if (has_allow(INVALID_SURROGATE)) { + if (hi >= 0x800) { + *dst++ = (u8)(0xE0 | (hi >> 12)); + *dst++ = (u8)(0x80 | ((hi >> 6) & 0x3F)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else if (hi >= 0x80) { + *dst++ = (u8)(0xC0 | (hi >> 6)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else { + *dst++ = (u8)hi; + } + *src_ptr = src; + *dst_ptr = dst; + return true; + } return_err(src - 6, "no low surrogate in string"); } if (unlikely(!hex_load_4(src + 2, &lo))) { + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + usize cnt = 0; + src += 2; /* skip \u */ + while (cnt < 4 && char_is_hex(src[cnt])) cnt++; + src += cnt; + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + *src_ptr = src; + *dst_ptr = dst; + return true; + } + if (has_allow(INVALID_SURROGATE)) { + if (hi >= 0x800) { + *dst++ = (u8)(0xE0 | (hi >> 12)); + *dst++ = (u8)(0x80 | ((hi >> 6) & 0x3F)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else if (hi >= 0x80) { + *dst++ = (u8)(0xC0 | (hi >> 6)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else { + *dst++ = (u8)hi; + } + *src_ptr = src; + *dst_ptr = dst; + return true; + } return_err(src - 6, "invalid escape in string"); } if (unlikely((lo & 0xFC00) != 0xDC00)) { + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + src += 6; + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + *src_ptr = src; + *dst_ptr = dst; + return true; + } + if (has_allow(INVALID_SURROGATE)) { + if (hi >= 0x800) { + *dst++ = (u8)(0xE0 | (hi >> 12)); + *dst++ = (u8)(0x80 | ((hi >> 6) & 0x3F)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else if (hi >= 0x80) { + *dst++ = (u8)(0xC0 | (hi >> 6)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else { + *dst++ = (u8)hi; + } + *src_ptr = src; + *dst_ptr = dst; + return true; + } return_err(src - 6, "invalid low surrogate in string"); } uni = ((((u32)hi - 0xD800) << 10) | @@ -4703,6 +4778,26 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, const char **msg) { *dst++ = (u8)(0x80 | ((uni >> 6) & 0x3F)); *dst++ = (u8)(0x80 | (uni & 0x3F)); src += 6; + } else { /* low surrogate without preceding high surrogate */ + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + *src_ptr = src; + *dst_ptr = dst; + return true; + } + if (!has_allow(INVALID_SURROGATE)) { + return_err(src - 6, "invalid low surrogate in string"); + } + if (hi >= 0x800) { + *dst++ = (u8)(0xE0 | (hi >> 12)); + *dst++ = (u8)(0x80 | ((hi >> 6) & 0x3F)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else if (hi >= 0x80) { + *dst++ = (u8)(0xC0 | (hi >> 6)); + *dst++ = (u8)(0x80 | (hi & 0x3F)); + } else { + *dst++ = (u8)hi; + } } *src_ptr = src; *dst_ptr = dst; @@ -4855,6 +4950,12 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, } #endif if (unlikely(pos == src)) { + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + dst = src; + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + ++src; + goto copy_utf8; + } if (has_allow(INVALID_UNICODE)) ++src; else return_err(src, "invalid UTF-8 encoding in string"); } @@ -4876,7 +4977,7 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, case 't': *dst++ = '\t'; src++; break; case 'u': src--; - if (!read_uni_esc(&src, &dst, msg)) return_err(src, *msg); + if (!read_uni_esc(&src, &dst, msg, flg)) return_err(src, *msg); break; default: { if (has_allow(EXT_ESCAPE)) { @@ -4935,11 +5036,17 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, if (con) con[0] = con[1] = NULL; return true; } else { - if (!has_allow(INVALID_UNICODE)) { - return_err(src, "unexpected control character in string"); + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + if (src >= eof) return_err(src, "unclosed string"); + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + src++; + } else { + if (!has_allow(INVALID_UNICODE)) { + return_err(src, "unexpected control character in string"); + } + if (src >= eof) return_err(src, "unclosed string"); + *dst++ = *src++; } - if (src >= eof) return_err(src, "unclosed string"); - *dst++ = *src++; } copy_ascii: @@ -5027,6 +5134,11 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, } #endif if (unlikely(pos == src)) { + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + ++src; + goto copy_utf8; + } if (!has_allow(INVALID_UNICODE)) { return_err(src, MSG_ERR_UTF8); } @@ -5131,7 +5243,7 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, dst = src; copy_escape: if (byte_match_2(src, "\\u")) { - if (!read_uni_esc(&src, &dst, msg)) return_err(src, *msg); + if (!read_uni_esc(&src, &dst, msg, flg)) return_err(src, *msg); } else { if (!char_is_id_next(*src)) return_suc(dst, src); return_err(src, "unexpected character in key"); @@ -5183,10 +5295,17 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, dst += 4; src += 4; } else { #if !YYJSON_DISABLE_UTF8_VALIDATION - if (!has_allow(INVALID_UNICODE)) return_err(src, MSG_ERR_UTF8); + if (!has_allow(INVALID_UNICODE) && + !has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) + return_err(src, MSG_ERR_UTF8); #endif - *dst = *src; - dst += 1; src += 1; + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + src += 1; + } else { + *dst = *src; + dst += 1; src += 1; + } } } if (char_is_id_ascii(*src)) goto copy_ascii; @@ -6206,6 +6325,13 @@ yyjson_doc *yyjson_read_opts(char *dat, usize len, } memset(eof, 0, YYJSON_PADDING_SIZE); + /* replacement has highest precedence: tolerate and replace all invalid + sequences so that the final output is always valid UTF-8 */ + if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + flg |= YYJSON_READ_ALLOW_INVALID_UNICODE; + flg |= YYJSON_READ_ALLOW_INVALID_SURROGATE; + } + if (has_allow(BOM)) { if (len >= 3 && is_utf8_bom(cur)) cur += 3; } @@ -6488,6 +6614,8 @@ yyjson_incr_state *yyjson_incr_new(char *buf, size_t buf_len, flg &= ~YYJSON_READ_JSON5; flg &= ~YYJSON_READ_ALLOW_BOM; flg &= ~YYJSON_READ_ALLOW_INVALID_UNICODE; + flg &= ~YYJSON_READ_ALLOW_INVALID_SURROGATE; + flg &= ~YYJSON_READ_REPLACE_INVALID_UNICODE; if (unlikely(!buf)) return NULL; if (unlikely(buf_len >= USIZE_MAX - YYJSON_PADDING_SIZE)) return NULL; diff --git a/src/yyjson.h b/src/yyjson.h index 5eb6d46..e95e11c 100644 --- a/src/yyjson.h +++ b/src/yyjson.h @@ -760,14 +760,13 @@ static const yyjson_read_flag YYJSON_READ_ALLOW_INF_AND_NAN = 1 << 4; inf/nan literal is also read as raw with `ALLOW_INF_AND_NAN` flag. */ static const yyjson_read_flag YYJSON_READ_NUMBER_AS_RAW = 1 << 5; -/** Allow reading invalid unicode when parsing string values (non-standard). - Invalid characters will be allowed to appear in the string values, but - invalid escape sequences will still be reported as errors. - This flag does not affect the performance of correctly encoded strings. - - @warning Strings in JSON values may contain incorrect encoding when this - option is used, you need to handle these strings carefully to avoid security - risks. */ +/** Allow reading raw invalid UTF-8 bytes in strings (non-standard). + This flag only affects raw bytes; `\uXXXX` escapes still require valid + surrogate pairs unless `YYJSON_READ_ALLOW_INVALID_SURROGATE` is also set. + Invalid escape sequences will still be reported as errors. + + @warning Strings may contain ill-formed UTF-8 when this option is used, + you need to handle these strings carefully to avoid security risks. */ static const yyjson_read_flag YYJSON_READ_ALLOW_INVALID_UNICODE = 1 << 6; /** Read big numbers as raw strings. These big numbers include integers that @@ -810,6 +809,22 @@ static const yyjson_read_flag YYJSON_READ_ALLOW_SINGLE_QUOTED_STR = 1 << 12; non-whitespace character with code point above `U+007F`. */ static const yyjson_read_flag YYJSON_READ_ALLOW_UNQUOTED_KEY = 1 << 13; +/** Replace invalid unicode code units with replacement character `U+FFFD` + when parsing string values (non-standard). + This flag implicitly enables `YYJSON_READ_ALLOW_INVALID_UNICODE` and + `YYJSON_READ_ALLOW_INVALID_SURROGATE`, so malformed input is tolerated + and replaced without needing those flags. + Note: when compiled with `YYJSON_DISABLE_UTF8_VALIDATION=ON`, invalid + UTF-8 byte sequences are not detected and therefore not replaced. */ +static const yyjson_read_flag YYJSON_READ_REPLACE_INVALID_UNICODE = 1 << 14; + +/** Allow unpaired surrogate code units in `\uXXXX` escapes (non-standard). + Raw invalid UTF-8 bytes are unaffected; use + `YYJSON_READ_ALLOW_INVALID_UNICODE` for that. When combined with + `YYJSON_READ_REPLACE_INVALID_UNICODE`, the surrogates are replaced with + `U+FFFD`. */ +static const yyjson_read_flag YYJSON_READ_ALLOW_INVALID_SURROGATE = 1 << 15; + /** Allow JSON5 format, see: [https://json5.org]. This flag supports all JSON5 features with some additional extensions: - Accepts more escape sequences than JSON5 (e.g. `\a`, `\e`). diff --git a/test/test_string.c b/test/test_string.c index d8080c3..2c69338 100644 --- a/test/test_string.c +++ b/test/test_string.c @@ -1450,6 +1450,39 @@ static void test_unquoted_key(void) { }, YYJSON_READ_ALLOW_INVALID_UNICODE); } +/*----------------------------------------------------------------------------*/ +/* MARK: - Invalid Unicode Flags */ +/*----------------------------------------------------------------------------*/ + +static void test_invalid_unicode_flags(void) { +#if !YYJSON_DISABLE_READER + /* unpaired surrogate */ + char inv_sur_hi[8 + YYJSON_PADDING_SIZE]; + memcpy(inv_sur_hi, "\"\\uD83D\"", 8); + memset(inv_sur_hi + 8, 0, YYJSON_PADDING_SIZE); + yyjson_doc *doc = yyjson_read(inv_sur_hi, 8, 0); + yy_assert(!doc); + + doc = yyjson_read(inv_sur_hi, 8, YYJSON_READ_ALLOW_INVALID_SURROGATE); + yy_assert(doc); + yyjson_val *val = yyjson_doc_get_root(doc); + const char *str = yyjson_get_str(val); + yy_assert(str && (u8)str[0] == 0xED && (u8)str[1] == 0xA0 && + (u8)str[2] == 0xBD && str[3] == '\0'); + yyjson_doc_free(doc); + + doc = yyjson_read(inv_sur_hi, 8, + YYJSON_READ_ALLOW_INVALID_SURROGATE | + YYJSON_READ_REPLACE_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && (u8)str[0] == 0xEF && (u8)str[1] == 0xBF && + (u8)str[2] == 0xBD && str[3] == '\0'); + yyjson_doc_free(doc); +#endif +} + /*============================================================================== @@ -1460,5 +1493,6 @@ yy_test_case(test_string) { test_read_write(); test_extended_escape(); test_single_quoted_string(); + test_invalid_unicode_flags(); test_unquoted_key(); } From 92a5d0bdf9c0051d3812e4677d76e6e69540fb44 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 17 Sep 2025 21:26:31 -0600 Subject: [PATCH 2/3] adjust internal API Signed-off-by: Eduardo Silva --- doc/API.md | 2 + src/yyjson.c | 178 ++++++++++++++++++++++++++------------------- src/yyjson.h | 9 ++- test/test_string.c | 82 +++++++++++++++++++++ 4 files changed, 197 insertions(+), 74 deletions(-) diff --git a/doc/API.md b/doc/API.md index 0b7a971..8b2e38c 100644 --- a/doc/API.md +++ b/doc/API.md @@ -758,12 +758,14 @@ Each JSON Value has a type and subtype, as specified in the table: | YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL | `double` number | | YYJSON_TYPE_STR | | String value | | YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC | String value, no-escape | +| YYJSON_TYPE_STR | YYJSON_SUBTYPE_UNIERR | String value, invalid unicode | | YYJSON_TYPE_ARR | | Array value | | YYJSON_TYPE_OBJ | | Object value | - `YYJSON_TYPE_NONE` means invalid value, it does not appear when the JSON is successfully parsed. - `YYJSON_TYPE_RAW` only appears when the corresponding flag `YYJSON_READ_XXX_AS_RAW` is used. - `YYJSON_SUBTYPE_NOESC` is used to optimize the writing speed of strings that do not need to be escaped. This subtype is used internally, and the user does not need to handle it. +- `YYJSON_SUBTYPE_UNIERR` marks strings that contained invalid Unicode during parsing. Such strings may require special handling. The following functions can be used to determine the type of a JSON value. diff --git a/src/yyjson.c b/src/yyjson.c index 25a5674..916b988 100644 --- a/src/yyjson.c +++ b/src/yyjson.c @@ -4658,7 +4658,8 @@ static_inline bool read_num(u8 **ptr, u8 **pre, yyjson_read_flag flg, /** Read unicode escape sequence. */ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, - const char **msg, yyjson_read_flag flg) { + const char **msg, yyjson_read_flag flg, + bool *unierr) { #define return_err(_end, _msg) *msg = _msg; *src_ptr = _end; return false u8 *src = *src_ptr; @@ -4669,12 +4670,19 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, src += 2; /* skip `\u` */ if (unlikely(!hex_load_4(src, &hi))) { if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { - usize cnt = 0; + usize cnt = 0, i; + u8 ch; while (cnt < 4 && char_is_hex(src[cnt])) cnt++; + ch = src[cnt]; + dst[0] = '\\'; + dst[1] = 'u'; + for (i = 0; i < cnt; i++) dst[2 + i] = src[i]; + dst += 2 + cnt; src += cnt; - *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; + if (ch && ch != '"' && ch != '\'') src++; *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } return_err(src - 2, "invalid escaped sequence in string"); @@ -4699,6 +4707,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } if (has_allow(INVALID_SURROGATE)) { @@ -4714,6 +4723,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, } *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } return_err(src - 6, "no low surrogate in string"); @@ -4727,6 +4737,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } if (has_allow(INVALID_SURROGATE)) { @@ -4742,6 +4753,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, } *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } return_err(src - 6, "invalid escape in string"); @@ -4752,6 +4764,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } if (has_allow(INVALID_SURROGATE)) { @@ -4767,6 +4780,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, } *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } return_err(src - 6, "invalid low surrogate in string"); @@ -4783,6 +4797,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; *src_ptr = src; *dst_ptr = dst; + if (unierr) *unierr = true; return true; } if (!has_allow(INVALID_SURROGATE)) { @@ -4798,6 +4813,7 @@ static_inline bool read_uni_esc(u8 **src_ptr, u8 **dst_ptr, } else { *dst++ = (u8)hi; } + if (unierr) *unierr = true; } *src_ptr = src; *dst_ptr = dst; @@ -4839,6 +4855,7 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, u8 *src = hdr, *dst = NULL, *pos; u16 hi, lo; u32 uni, tmp; + bool unierr = false; /* Resume incremental parsing. */ if (con && unlikely(con[0])) { @@ -4901,7 +4918,8 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, gcc_store_barrier(*src); if (likely(*src == quo)) { val->tag = ((u64)(src - hdr) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR | - (quo == '"' ? YYJSON_SUBTYPE_NOESC : 0); + (unierr ? YYJSON_SUBTYPE_UNIERR : + (quo == '"' ? YYJSON_SUBTYPE_NOESC : 0)); val->uni.str = (const char *)hdr; *src = '\0'; *end = src + 1; @@ -4950,14 +4968,13 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, } #endif if (unlikely(pos == src)) { - if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { - dst = src; - *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; - ++src; - goto copy_utf8; - } - if (has_allow(INVALID_UNICODE)) ++src; - else return_err(src, "invalid UTF-8 encoding in string"); + if (!has_allow(INVALID_UNICODE) && + !has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) + return_err(src, "invalid UTF-8 encoding in string"); + dst = src; + *dst++ = *src++; + unierr = true; + goto copy_utf8; } goto skip_ascii; } @@ -4977,7 +4994,7 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, case 't': *dst++ = '\t'; src++; break; case 'u': src--; - if (!read_uni_esc(&src, &dst, msg, flg)) return_err(src, *msg); + if (!read_uni_esc(&src, &dst, msg, flg, &unierr)) return_err(src, *msg); break; default: { if (has_allow(EXT_ESCAPE)) { @@ -5029,24 +5046,21 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, } } } else if (likely(*src == quo)) { - val->tag = ((u64)(dst - hdr) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR; + val->tag = ((u64)(dst - hdr) << YYJSON_TAG_BIT) | YYJSON_TYPE_STR | + (unierr ? YYJSON_SUBTYPE_UNIERR : 0); val->uni.str = (const char *)hdr; *dst = '\0'; *end = src + 1; if (con) con[0] = con[1] = NULL; return true; } else { - if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { - if (src >= eof) return_err(src, "unclosed string"); - *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; - src++; - } else { - if (!has_allow(INVALID_UNICODE)) { - return_err(src, "unexpected control character in string"); - } - if (src >= eof) return_err(src, "unclosed string"); - *dst++ = *src++; + if (!has_allow(INVALID_UNICODE) && + !has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { + return_err(src, "unexpected control character in string"); } + if (src >= eof) return_err(src, "unclosed string"); + *dst++ = *src++; + unierr = true; } copy_ascii: @@ -5134,15 +5148,12 @@ static_inline bool read_str_opt(u8 quo, u8 **ptr, u8 *eof, yyjson_read_flag flg, } #endif if (unlikely(pos == src)) { - if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { - *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; - ++src; - goto copy_utf8; - } - if (!has_allow(INVALID_UNICODE)) { + if (!has_allow(INVALID_UNICODE) && + !has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) return_err(src, MSG_ERR_UTF8); - } - goto copy_ascii_stop_1; + *dst++ = *src++; + unierr = true; + goto copy_utf8; } goto copy_ascii; } @@ -5177,7 +5188,7 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, #define return_suc(_str_end, _cur_end) do { \ val->tag = ((u64)(_str_end - hdr) << YYJSON_TAG_BIT) | \ - (u64)(YYJSON_TYPE_STR); \ + (u64)(YYJSON_TYPE_STR | (unierr ? YYJSON_SUBTYPE_UNIERR : 0)); \ val->uni.str = (const char *)hdr; \ *pre = _str_end; *end = _cur_end; \ return true; \ @@ -5188,6 +5199,7 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, u8 *src = hdr, *dst = NULL; u16 hi, lo; u32 uni, tmp; + bool unierr = false; /* add null-terminator for previous raw string */ **pre = '\0'; @@ -5232,9 +5244,14 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, src += 4; } else { #if !YYJSON_DISABLE_UTF8_VALIDATION - if (!has_allow(INVALID_UNICODE)) return_err(src, MSG_ERR_UTF8); + if (!has_allow(INVALID_UNICODE) && + !has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) + return_err(src, MSG_ERR_UTF8); #endif - src += 1; + dst = src; + *dst++ = *src++; + unierr = true; + goto copy_utf8; } } if (char_is_id_ascii(*src)) goto skip_ascii; @@ -5243,7 +5260,7 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, dst = src; copy_escape: if (byte_match_2(src, "\\u")) { - if (!read_uni_esc(&src, &dst, msg, flg)) return_err(src, *msg); + if (!read_uni_esc(&src, &dst, msg, flg, &unierr)) return_err(src, *msg); } else { if (!char_is_id_next(*src)) return_suc(dst, src); return_err(src, "unexpected character in key"); @@ -5299,13 +5316,8 @@ static_noinline bool read_str_id(u8 **ptr, u8 *eof, yyjson_read_flag flg, !has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) return_err(src, MSG_ERR_UTF8); #endif - if (has_rflag(flg, YYJSON_READ_REPLACE_INVALID_UNICODE, 1)) { - *dst++ = 0xEF; *dst++ = 0xBF; *dst++ = 0xBD; - src += 1; - } else { - *dst = *src; - dst += 1; src += 1; - } + *dst++ = *src++; + unierr = true; } } if (char_is_id_ascii(*src)) goto copy_ascii; @@ -9120,12 +9132,16 @@ static_inline u8 *yyjson_write_single(yyjson_val *val, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 2 + end_len); - if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { - cur = write_str_noesc(cur, str_ptr, str_len); - } else { - cur = write_str(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; - } + do { + yyjson_subtype st = unsafe_yyjson_get_subtype(val); + if (likely(cpy) && st == YYJSON_SUBTYPE_NOESC) { + cur = write_str_noesc(cur, str_ptr, str_len); + } else { + bool inv2 = inv || (st == YYJSON_SUBTYPE_UNIERR); + cur = write_str(cur, esc, inv2, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } + } while (0); break; case YYJSON_TYPE_NUM: @@ -9257,12 +9273,16 @@ static_inline u8 *yyjson_write_minify(const yyjson_val *root, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 16); - if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { - cur = write_str_noesc(cur, str_ptr, str_len); - } else { - cur = write_str(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; - } + do { + yyjson_subtype st = unsafe_yyjson_get_subtype(val); + if (likely(cpy) && st == YYJSON_SUBTYPE_NOESC) { + cur = write_str_noesc(cur, str_ptr, str_len); + } else { + bool inv2 = inv || (st == YYJSON_SUBTYPE_UNIERR); + cur = write_str(cur, esc, inv2, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } + } while (0); *cur++ = is_key ? ':' : ','; goto val_end; } @@ -9443,12 +9463,16 @@ static_inline u8 *yyjson_write_pretty(const yyjson_val *root, check_str_len(str_len); incr_len(str_len * 6 + 16 + (no_indent ? 0 : level * 4)); cur = write_indent(cur, no_indent ? 0 : level, spaces); - if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { - cur = write_str_noesc(cur, str_ptr, str_len); - } else { - cur = write_str(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; - } + do { + yyjson_subtype st = unsafe_yyjson_get_subtype(val); + if (likely(cpy) && st == YYJSON_SUBTYPE_NOESC) { + cur = write_str_noesc(cur, str_ptr, str_len); + } else { + bool inv2 = inv || (st == YYJSON_SUBTYPE_UNIERR); + cur = write_str(cur, esc, inv2, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } + } while (0); *cur++ = is_key ? ':' : ','; *cur++ = is_key ? ' ' : '\n'; goto val_end; @@ -9812,12 +9836,16 @@ static_inline u8 *yyjson_mut_write_minify(const yyjson_mut_val *root, str_ptr = (const u8 *)unsafe_yyjson_get_str(val); check_str_len(str_len); incr_len(str_len * 6 + 16); - if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { - cur = write_str_noesc(cur, str_ptr, str_len); - } else { - cur = write_str(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; - } + do { + yyjson_subtype st = unsafe_yyjson_get_subtype(val); + if (likely(cpy) && st == YYJSON_SUBTYPE_NOESC) { + cur = write_str_noesc(cur, str_ptr, str_len); + } else { + bool inv2 = inv || (st == YYJSON_SUBTYPE_UNIERR); + cur = write_str(cur, esc, inv2, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } + } while (0); *cur++ = is_key ? ':' : ','; goto val_end; } @@ -10004,12 +10032,16 @@ static_inline u8 *yyjson_mut_write_pretty(const yyjson_mut_val *root, check_str_len(str_len); incr_len(str_len * 6 + 16 + (no_indent ? 0 : level * 4)); cur = write_indent(cur, no_indent ? 0 : level, spaces); - if (likely(cpy) && unsafe_yyjson_get_subtype(val)) { - cur = write_str_noesc(cur, str_ptr, str_len); - } else { - cur = write_str(cur, esc, inv, str_ptr, str_len, enc_table); - if (unlikely(!cur)) goto fail_str; - } + do { + yyjson_subtype st = unsafe_yyjson_get_subtype(val); + if (likely(cpy) && st == YYJSON_SUBTYPE_NOESC) { + cur = write_str_noesc(cur, str_ptr, str_len); + } else { + bool inv2 = inv || (st == YYJSON_SUBTYPE_UNIERR); + cur = write_str(cur, esc, inv2, str_ptr, str_len, enc_table); + if (unlikely(!cur)) goto fail_str; + } + } while (0); *cur++ = is_key ? ':' : ','; *cur++ = is_key ? ' ' : '\n'; goto val_end; diff --git a/src/yyjson.h b/src/yyjson.h index e95e11c..5b62844 100644 --- a/src/yyjson.h +++ b/src/yyjson.h @@ -527,7 +527,7 @@ typedef uint8_t yyjson_type; #define YYJSON_TYPE_BOOL ((uint8_t)3) /* _____011 */ /** Number type, subtype: UINT, SINT, REAL. */ #define YYJSON_TYPE_NUM ((uint8_t)4) /* _____100 */ -/** String type, subtype: NONE, NOESC. */ +/** String type, subtype: NONE, NOESC, UNIERR. */ #define YYJSON_TYPE_STR ((uint8_t)5) /* _____101 */ /** Array type, no subtype. */ #define YYJSON_TYPE_ARR ((uint8_t)6) /* _____110 */ @@ -550,6 +550,8 @@ typedef uint8_t yyjson_subtype; #define YYJSON_SUBTYPE_REAL ((uint8_t)(2 << 3)) /* ___10___ */ /** String that do not need to be escaped for writing (internal use). */ #define YYJSON_SUBTYPE_NOESC ((uint8_t)(1 << 3)) /* ___01___ */ +/** String containing invalid Unicode sequences. */ +#define YYJSON_SUBTYPE_UNIERR ((uint8_t)(2 << 3)) /* ___10___ */ /** The mask used to extract the type of a JSON value. */ #define YYJSON_TYPE_MASK ((uint8_t)0x07) /* _____111 */ @@ -811,6 +813,10 @@ static const yyjson_read_flag YYJSON_READ_ALLOW_UNQUOTED_KEY = 1 << 13; /** Replace invalid unicode code units with replacement character `U+FFFD` when parsing string values (non-standard). + Invalid UTF-8 byte sequences that are shorter than three bytes cannot be + expanded in-place and are left unchanged. Strings that still contain + invalid bytes are marked with `YYJSON_SUBTYPE_UNIERR` for the caller to + detect. This flag implicitly enables `YYJSON_READ_ALLOW_INVALID_UNICODE` and `YYJSON_READ_ALLOW_INVALID_SURROGATE`, so malformed input is tolerated and replaced without needing those flags. @@ -5237,6 +5243,7 @@ yyjson_api_inline const char *yyjson_get_type_desc(yyjson_val *val) { case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE: return "null"; case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE: return "string"; case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC: return "string"; + case YYJSON_TYPE_STR | YYJSON_SUBTYPE_UNIERR: return "string"; case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE: return "array"; case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE: return "object"; case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_TRUE: return "true"; diff --git a/test/test_string.c b/test/test_string.c index 2c69338..670ba69 100644 --- a/test/test_string.c +++ b/test/test_string.c @@ -1469,6 +1469,7 @@ static void test_invalid_unicode_flags(void) { const char *str = yyjson_get_str(val); yy_assert(str && (u8)str[0] == 0xED && (u8)str[1] == 0xA0 && (u8)str[2] == 0xBD && str[3] == '\0'); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); yyjson_doc_free(doc); doc = yyjson_read(inv_sur_hi, 8, @@ -1479,6 +1480,87 @@ static void test_invalid_unicode_flags(void) { str = yyjson_get_str(val); yy_assert(str && (u8)str[0] == 0xEF && (u8)str[1] == 0xBF && (u8)str[2] == 0xBD && str[3] == '\0'); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); + yyjson_doc_free(doc); + + /* invalid UTF-8 byte */ + char inv_utf8[4 + YYJSON_PADDING_SIZE]; + inv_utf8[0] = '"'; + inv_utf8[1] = (char)0x80; + inv_utf8[2] = '"'; + inv_utf8[3] = '\0'; + memset(inv_utf8 + 3, 0, YYJSON_PADDING_SIZE); + + yy_assert(!yyjson_read(inv_utf8, 3, 0)); + + doc = yyjson_read(inv_utf8, 3, YYJSON_READ_ALLOW_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && (u8)str[0] == 0x80 && str[1] == '\0'); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); + yyjson_doc_free(doc); + + doc = yyjson_read(inv_utf8, 3, YYJSON_READ_REPLACE_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && (u8)str[0] == 0x80 && str[1] == '\0'); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); + yyjson_doc_free(doc); + + /* malformed \u with no digits */ + char inv_esc1[4 + YYJSON_PADDING_SIZE]; + memcpy(inv_esc1, "\"\\u\"", 4); + memset(inv_esc1 + 4, 0, YYJSON_PADDING_SIZE); + doc = yyjson_read(inv_esc1, 4, YYJSON_READ_REPLACE_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && str[0] == '\\' && str[1] == 'u' && str[2] == '\0'); + yy_assert(yyjson_get_len(val) == 2); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); + yyjson_doc_free(doc); + + /* malformed \u with non-hex char */ + char inv_esc2[5 + YYJSON_PADDING_SIZE]; + memcpy(inv_esc2, "\"\\uX\"", 5); + memset(inv_esc2 + 5, 0, YYJSON_PADDING_SIZE); + doc = yyjson_read(inv_esc2, 5, YYJSON_READ_REPLACE_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && str[0] == '\\' && str[1] == 'u' && str[2] == '\0'); + yy_assert(yyjson_get_len(val) == 2); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); + yyjson_doc_free(doc); + + /* malformed \u with truncated trailing hex digits */ + char inv_esc3[8 + YYJSON_PADDING_SIZE]; + memcpy(inv_esc3, "\"\\u123G\"", 8); + memset(inv_esc3 + 8, 0, YYJSON_PADDING_SIZE); + doc = yyjson_read(inv_esc3, 8, YYJSON_READ_REPLACE_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && str[0] == '\\' && str[1] == 'u' && + str[2] == '1' && str[3] == '2' && str[4] == '3' && str[5] == '\0'); + yy_assert(yyjson_get_len(val) == 5); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); + yyjson_doc_free(doc); + + /* malformed surrogate pair with incomplete low surrogate */ + char inv_sur_pair[10 + YYJSON_PADDING_SIZE]; + memcpy(inv_sur_pair, "\"\\uD800\\u\"", 10); + memset(inv_sur_pair + 10, 0, YYJSON_PADDING_SIZE); + doc = yyjson_read(inv_sur_pair, 10, YYJSON_READ_REPLACE_INVALID_UNICODE); + yy_assert(doc); + val = yyjson_doc_get_root(doc); + str = yyjson_get_str(val); + yy_assert(str && (u8)str[0] == 0xEF && (u8)str[1] == 0xBF && + (u8)str[2] == 0xBD && str[3] == '\0'); + yy_assert(yyjson_get_len(val) == 3); + yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); yyjson_doc_free(doc); #endif } From 825aa376966dadea756286dfa43cc8106fe2ea77 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sun, 21 Sep 2025 09:12:48 -0600 Subject: [PATCH 3/3] test: string: fix handling of test_invalid_unicode_flags Signed-off-by: Eduardo Silva --- test/test_string.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_string.c b/test/test_string.c index 670ba69..2b6c082 100644 --- a/test/test_string.c +++ b/test/test_string.c @@ -1455,7 +1455,7 @@ static void test_unquoted_key(void) { /*----------------------------------------------------------------------------*/ static void test_invalid_unicode_flags(void) { -#if !YYJSON_DISABLE_READER +#if !YYJSON_DISABLE_READER && !YYJSON_DISABLE_NON_STANDARD /* unpaired surrogate */ char inv_sur_hi[8 + YYJSON_PADDING_SIZE]; memcpy(inv_sur_hi, "\"\\uD83D\"", 8); @@ -1562,7 +1562,7 @@ static void test_invalid_unicode_flags(void) { yy_assert(yyjson_get_len(val) == 3); yy_assert(yyjson_get_subtype(val) == YYJSON_SUBTYPE_UNIERR); yyjson_doc_free(doc); -#endif +#endif /* !YYJSON_DISABLE_READER && !YYJSON_DISABLE_NON_STANDARD */ }