Add failing tests for non-BMP text round-tripping (#25)

StringUtilities::ToUtf8/FromUtf8 use std::codecvt_utf8<wchar_t>, which converts
UTF-8 to and from UCS-2 when wchar_t is 16 bits. Code points above U+FFFF
therefore do not round-trip on Windows, while the same code is correct on Linux
and macOS where wchar_t is 32 bits.

These tests reproduce that: they are expected to fail on Windows and pass on
Linux and macOS. That asymmetry is the signature of the defect - if the tests
were to fail everywhere they would be measuring something else, and if they were
to pass everywhere they would not reproduce the bug at all.

Every wide literal uses universal-character escapes instead of literal non-ASCII
source bytes. MSVC is not passed /utf-8, so it decodes such bytes in the system
codepage; the pre-existing Greek literals in database_test.cc compare a mangled
literal against the same mangled literal and so pass without testing anything.
Expected UTF-8 is asserted as exact bytes, and the end-to-end test matches on
hex(name) in SQL, so a conversion that is wrong in both directions cannot pass.

Tests reach src/internal/string_utilities.h via a new include directory, so the
conversion can be exercised directly and not only through Save/Fetch.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: jkalias

tests/CMakeLists.txt

@@ -8,6 +8,10 @@ include(GoogleTest)
 include_directories ("${PROJECT_SOURCE_DIR}/include")
 include_directories ("./include")
 
+# Internal headers (src/internal/...) so the UTF-8 conversion helpers can be tested directly
+# rather than only through the public wstring API
+include_directories ("${PROJECT_SOURCE_DIR}/src")
+
 # Collect test sources into the variable TEST_SOURCES
 file (GLOB TEST_SOURCES
       "*.cpp"

tests/database_test.cc

@@ -874,3 +874,26 @@ TEST_F(DatabaseTest, RawSqlQueryForPersistedRecord) {
     EXPECT_EQ(52, fetched_persons[0].id);
     EXPECT_EQ(L"johnie", fetched_persons[0].first_name);
 }
+
+TEST_F(DatabaseTest, TextOutsideTheBasicMultilingualPlaneSurvivesSaveAndFetch) {
+    const auto db = Database::Instance();
+
+    // Code points above U+FFFF need a surrogate pair where wchar_t is 16 bits (Windows) and a
+    // single code unit where it is 32 bits (Linux, macOS). The stored UTF-8 and the fetched
+    // wstring must be identical on every platform. Written with universal-character escapes
+    // so the test does not depend on how the compiler decodes this file's source bytes.
+    const std::wstring name = L"\U0001F600";
+    const std::wstring address = L"\U00010000\U0010FFFF";
+
+    db->Save(Company{name, 30, address, 50000.0, 1});
+
+    const auto fetched = db->Fetch<Company>(1);
+    EXPECT_EQ(name, fetched.name);
+    EXPECT_EQ(address, fetched.address);
+
+    // The comparisons above are self-consistent: a conversion wrong in both directions would
+    // satisfy them. Match on the stored bytes instead - U+1F600 is F0 9F 98 80 in UTF-8 - so
+    // the row only disappears if what SQLite actually holds is correct UTF-8.
+    db->UnsafeSql("DELETE FROM Company WHERE hex(name) = 'F09F9880'");
+    EXPECT_EQ(0, db->FetchAll<Company>().size());
+}

tests/utf8_test.cc

@@ -0,0 +1,125 @@
+// MIT License
+//
+// Copyright (c) 2026 Ioannis Kaliakatsos
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "internal/string_utilities.h"
+
+using namespace sqlite_reflection;
+
+// Every wide literal in this file is written with universal-character escapes rather than
+// literal non-ASCII source bytes. MSVC is not passed /utf-8, so it decodes non-ASCII source
+// bytes in the system codepage; a test written with literal characters would compare a
+// mangled literal against the same mangled literal and pass while proving nothing. Escapes
+// are charset-independent, and the expected UTF-8 is asserted as exact bytes rather than by
+// round-tripping, so a conversion that is wrong in both directions cannot pass either.
+
+namespace {
+// A code point is encoded as a surrogate pair in a 16-bit wchar_t (Windows) and as a single
+// code unit in a 32-bit wchar_t (Linux, macOS). The expected UTF-8 is identical on both.
+const char* const kEmojiUtf8 = "\xF0\x9F\x98\x80";               // U+1F600 GRINNING FACE
+const char* const kFirstSupplementaryUtf8 = "\xF0\x90\x80\x80";  // U+10000, first non-BMP
+const char* const kLastCodePointUtf8 = "\xF4\x8F\xBF\xBF";       // U+10FFFF, highest valid
+const char* const kLastBmpUtf8 = "\xEF\xBF\xBF";                 // U+FFFF, last BMP code point
+
+std::wstring FromUtf8(const std::string& utf8) {
+    return StringUtilities::FromUtf8(utf8.data(), utf8.size());
+}
+}  // namespace
+
+TEST(Utf8Test, EncodesAsciiToExactBytes) {
+    EXPECT_EQ(std::string("Appleseed"), StringUtilities::ToUtf8(L"Appleseed"));
+}
+
+TEST(Utf8Test, EncodesEmptyString) {
+    EXPECT_EQ(std::string(), StringUtilities::ToUtf8(std::wstring()));
+}
+
+TEST(Utf8Test, EncodesBmpToExactBytes) {
+    // U+03C0 GREEK SMALL LETTER PI, U+03B1 GREEK SMALL LETTER ALPHA: two-byte sequences.
+    EXPECT_EQ(std::string("\xCF\x80\xCE\xB1"), StringUtilities::ToUtf8(L"\u03C0\u03B1"));
+}
+
+TEST(Utf8Test, EncodesLastBmpCodePointToExactBytes) {
+    EXPECT_EQ(std::string(kLastBmpUtf8), StringUtilities::ToUtf8(L"\uFFFF"));
+}
+
+TEST(Utf8Test, EncodesFirstSupplementaryCodePointToExactBytes) {
+    // U+10000 is the low side of the surrogate boundary: the first code point that needs a
+    // surrogate pair in UTF-16 and a four-byte UTF-8 sequence.
+    EXPECT_EQ(std::string(kFirstSupplementaryUtf8), StringUtilities::ToUtf8(L"\U00010000"));
+}
+
+TEST(Utf8Test, EncodesEmojiToExactBytes) {
+    EXPECT_EQ(std::string(kEmojiUtf8), StringUtilities::ToUtf8(L"\U0001F600"));
+}
+
+TEST(Utf8Test, EncodesLastValidCodePointToExactBytes) {
+    EXPECT_EQ(std::string(kLastCodePointUtf8), StringUtilities::ToUtf8(L"\U0010FFFF"));
+}
+
+TEST(Utf8Test, EncodesSupplementaryCodePointMixedWithAscii) {
+    EXPECT_EQ(std::string("a") + kEmojiUtf8 + "b", StringUtilities::ToUtf8(L"a\U0001F600b"));
+}
+
+TEST(Utf8Test, DecodesAscii) {
+    EXPECT_EQ(std::wstring(L"Appleseed"), FromUtf8("Appleseed"));
+}
+
+TEST(Utf8Test, DecodesEmptyString) {
+    EXPECT_EQ(std::wstring(), FromUtf8(std::string()));
+}
+
+TEST(Utf8Test, DecodesBmp) {
+    EXPECT_EQ(std::wstring(L"\u03C0\u03B1"), FromUtf8("\xCF\x80\xCE\xB1"));
+}
+
+TEST(Utf8Test, DecodesLastBmpCodePoint) {
+    EXPECT_EQ(std::wstring(L"\uFFFF"), FromUtf8(kLastBmpUtf8));
+}
+
+TEST(Utf8Test, DecodesFirstSupplementaryCodePoint) {
+    EXPECT_EQ(std::wstring(L"\U00010000"), FromUtf8(kFirstSupplementaryUtf8));
+}
+
+TEST(Utf8Test, DecodesEmoji) {
+    EXPECT_EQ(std::wstring(L"\U0001F600"), FromUtf8(kEmojiUtf8));
+}
+
+TEST(Utf8Test, DecodesLastValidCodePoint) {
+    EXPECT_EQ(std::wstring(L"\U0010FFFF"), FromUtf8(kLastCodePointUtf8));
+}
+
+TEST(Utf8Test, RoundTripsSupplementaryCodePoints) {
+    const std::wstring original = L"\U0001F600\U00010000\U0010FFFF";
+    EXPECT_EQ(original, FromUtf8(StringUtilities::ToUtf8(original)));
+}
+
+TEST(Utf8Test, SupplementaryCodePointSurvivesLengthAndContent) {
+    // Guards against a conversion that silently drops or truncates the pair rather than
+    // producing a wrong-but-present result.
+    const std::wstring decoded = FromUtf8(kEmojiUtf8);
+    EXPECT_FALSE(decoded.empty());
+    EXPECT_EQ(std::wstring(L"\U0001F600"), decoded);
+}