From 16b0853654c96df222792f754996d86e2757cf4f Mon Sep 17 00:00:00 2001 From: Hoang Manh Linh Date: Thu, 18 Jun 2026 17:00:01 +0700 Subject: [PATCH] fix(epub): NFC-normalize EPUB text so NFD diacritics render correctly (#2277) Co-authored-by: Uri Tauber Co-authored-by: Julia --- lib/Epub/Epub.cpp | 6 +- lib/Epub/Epub/BookMetadataCache.cpp | 7 +- lib/Epub/Epub/ParsedText.cpp | 8 + lib/Epub/Epub/Section.cpp | 3 +- lib/Utf8/Utf8.cpp | 68 ++++++++ lib/Utf8/Utf8.h | 6 + lib/Utf8/Utf8ComposeTable.h | 229 ++++++++++++++++++++++++++ test/CMakeLists.txt | 1 + test/utf8_compose/CMakeLists.txt | 15 ++ test/utf8_compose/Utf8ComposeTest.cpp | 54 ++++++ 10 files changed, 392 insertions(+), 5 deletions(-) create mode 100644 lib/Utf8/Utf8ComposeTable.h create mode 100644 test/utf8_compose/CMakeLists.txt create mode 100644 test/utf8_compose/Utf8ComposeTest.cpp diff --git a/lib/Epub/Epub.cpp b/lib/Epub/Epub.cpp index 47d11708..5f5dcc8a 100644 --- a/lib/Epub/Epub.cpp +++ b/lib/Epub/Epub.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "Epub/parsers/ContainerParser.h" @@ -73,8 +74,9 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, const return false; } - // Grab data from opfParser into epub - bookMetadata.title = opfParser.title; + // Grab data from opfParser into epub. Normalize titles to NFC so NFD (combining + // mark) text renders correctly — the device fonts have no mark positioning. + bookMetadata.title = utf8ComposeNfc(opfParser.title); bookMetadata.author = opfParser.author; bookMetadata.language = opfParser.language; bookMetadata.coverItemHref = opfParser.coverItemHref; diff --git a/lib/Epub/Epub/BookMetadataCache.cpp b/lib/Epub/Epub/BookMetadataCache.cpp index 149cc1d8..76d75fa2 100644 --- a/lib/Epub/Epub/BookMetadataCache.cpp +++ b/lib/Epub/Epub/BookMetadataCache.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -9,7 +10,7 @@ #include "FsHelpers.h" namespace { -constexpr uint8_t BOOK_CACHE_VERSION = 7; +constexpr uint8_t BOOK_CACHE_VERSION = 8; // v8: TOC/book titles stored NFC-composed constexpr char bookBinFile[] = "/book.bin"; constexpr char tmpSpineBinFile[] = "/spine.bin.tmp"; constexpr char tmpTocBinFile[] = "/toc.bin.tmp"; @@ -364,7 +365,9 @@ void BookMetadataCache::createTocEntry(const std::string& title, const std::stri } } - const TocEntry entry(title, href, anchor, level, spineIndex); + // Compose the title to NFC at index time so the cache stores precomposed glyphs; + // device fonts have no combining-mark positioning, so NFD titles render broken. + const TocEntry entry(utf8ComposeNfc(title), href, anchor, level, spineIndex); writeTocEntry(tocFile, entry); tocCount++; } diff --git a/lib/Epub/Epub/ParsedText.cpp b/lib/Epub/Epub/ParsedText.cpp index 124a857d..e57d1743 100644 --- a/lib/Epub/Epub/ParsedText.cpp +++ b/lib/Epub/Epub/ParsedText.cpp @@ -254,6 +254,14 @@ void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle, const bool attachToPrevious) { if (word.empty()) return; + // The device fonts carry no combining-mark positioning, so EPUB text stored in NFD + // (a base letter followed by separate combining accents -- common for Vietnamese, + // and used for many EPUB

chapter headings) renders with the marks detached or + // misplaced. Compose to NFC here, the single funnel every word passes through, so a + // precomposed glyph is used instead. This runs once per word at layout time (the + // result is cached in the section file) and is a cheap no-op for mark-free text. + word = utf8ComposeNfc(word); + EpdFontFamily::Style baseStyle = fontStyle; if (underline) { baseStyle = static_cast(baseStyle | EpdFontFamily::UNDERLINE); diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index c13da526..db249481 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -10,7 +10,8 @@ #include "parsers/ChapterHtmlSlimParser.h" namespace { -constexpr uint8_t SECTION_FILE_VERSION = 26; +// v27: words NFC-composed at layout time; bump invalidates NFD section caches. +constexpr uint8_t SECTION_FILE_VERSION = 27; constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(uint8_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) + sizeof(bool) + sizeof(uint8_t) + sizeof(bool) + sizeof(uint32_t) + sizeof(uint32_t) + diff --git a/lib/Utf8/Utf8.cpp b/lib/Utf8/Utf8.cpp index b59a14d4..340c4510 100644 --- a/lib/Utf8/Utf8.cpp +++ b/lib/Utf8/Utf8.cpp @@ -1,5 +1,73 @@ #include "Utf8.h" +#include "Utf8ComposeTable.h" + +namespace { +// Look up the canonical composition of (base + combining mark), or 0 if none. +uint32_t utf8ComposePair(const uint32_t base, const uint32_t mark) { + if (base > 0xFFFF || mark > 0xFFFF) return 0; + int lo = 0; + int hi = kUtf8ComposeTableSize - 1; + while (lo <= hi) { + const int mid = (lo + hi) / 2; + const Utf8ComposeEntry& e = kUtf8ComposeTable[mid]; + if (e.base < base || (e.base == base && e.mark < mark)) { + lo = mid + 1; + } else if (e.base > base || (e.base == base && e.mark > mark)) { + hi = mid - 1; + } else { + return e.composed; + } + } + return 0; +} +} // namespace + +std::string utf8ComposeNfc(const std::string& in) { + // Fast path: NFC composition can only change text that contains a combining + // diacritical mark U+0300-036F (UTF-8 lead byte 0xCC or 0xCD). Plain ASCII and + // already-precomposed (NFC) text -- the vast majority of words -- have none, so + // return them untouched without walking codepoints or allocating. A 0xCD that is + // actually a non-combining codepoint just falls through to the full pass below. + bool maybeHasMarks = false; + for (const unsigned char c : in) { + if (c == 0xCC || c == 0xCD) { + maybeHasMarks = true; + break; + } + } + if (!maybeHasMarks) return in; + + std::string out; + out.reserve(in.size()); + const unsigned char* p = reinterpret_cast(in.c_str()); + uint32_t base = 0; + bool haveBase = false; + while (*p) { + const uint32_t cp = utf8NextCodepoint(&p); + if (cp == 0) break; + if (utf8IsCombiningMark(cp)) { + const uint32_t composed = haveBase ? utf8ComposePair(base, cp) : 0; + if (composed) { + base = composed; // keep accumulating further marks onto the composed char + continue; + } + // No composition: flush the pending base, then emit the mark unchanged. + if (haveBase) { + utf8AppendCodepoint(base, out); + haveBase = false; + } + utf8AppendCodepoint(cp, out); + } else { + if (haveBase) utf8AppendCodepoint(base, out); + base = cp; + haveBase = true; + } + } + if (haveBase) utf8AppendCodepoint(base, out); + return out; +} + int utf8CodepointLen(const unsigned char c) { if (c < 0x80) return 1; // 0xxxxxxx if ((c >> 5) == 0x6) return 2; // 110xxxxx diff --git a/lib/Utf8/Utf8.h b/lib/Utf8/Utf8.h index 2d3375fd..64c04808 100644 --- a/lib/Utf8/Utf8.h +++ b/lib/Utf8/Utf8.h @@ -12,6 +12,12 @@ size_t utf8RemoveLastChar(std::string& str); // Truncate string by removing N UTF-8 codepoints from the end. void utf8TruncateChars(std::string& str, size_t numChars); +// Canonical composition (NFC) for the Latin / Vietnamese range: precomposes a +// base letter followed by combining diacritical mark(s) into a single codepoint. +// Needed because the device fonts have no combining-mark positioning, so text +// stored in NFD (e.g. some EPUB chapter titles) otherwise renders broken. +std::string utf8ComposeNfc(const std::string& in); + // Truncate a raw char buffer to the last complete UTF-8 codepoint boundary. // Returns the new length (<= len). If the buffer ends mid-sequence, the // incomplete trailing bytes are excluded. diff --git a/lib/Utf8/Utf8ComposeTable.h b/lib/Utf8/Utf8ComposeTable.h new file mode 100644 index 00000000..4b7ee00e --- /dev/null +++ b/lib/Utf8/Utf8ComposeTable.h @@ -0,0 +1,229 @@ +// Auto-generated canonical composition (NFC) table for the Latin / Vietnamese +// range (combining marks U+0300-U+036F). Generated from Python unicodedata. +// Used by utf8ComposeNfc() to precompose decomposed (NFD) text so the device +// fonts (which lack combining-mark positioning) render it correctly. +#pragma once +#include + +struct Utf8ComposeEntry { + uint16_t base; + uint16_t mark; + uint16_t composed; +}; + +// Sorted by (base, mark) for binary search. +static constexpr Utf8ComposeEntry kUtf8ComposeTable[] = { + {0x0041, 0x0300, 0x00C0}, {0x0041, 0x0301, 0x00C1}, {0x0041, 0x0302, 0x00C2}, {0x0041, 0x0303, 0x00C3}, + {0x0041, 0x0304, 0x0100}, {0x0041, 0x0306, 0x0102}, {0x0041, 0x0307, 0x0226}, {0x0041, 0x0308, 0x00C4}, + {0x0041, 0x0309, 0x1EA2}, {0x0041, 0x030A, 0x00C5}, {0x0041, 0x030C, 0x01CD}, {0x0041, 0x030F, 0x0200}, + {0x0041, 0x0311, 0x0202}, {0x0041, 0x0323, 0x1EA0}, {0x0041, 0x0325, 0x1E00}, {0x0041, 0x0328, 0x0104}, + {0x0041, 0x0340, 0x00C0}, {0x0041, 0x0341, 0x00C1}, {0x0042, 0x0307, 0x1E02}, {0x0042, 0x0323, 0x1E04}, + {0x0042, 0x0331, 0x1E06}, {0x0043, 0x0301, 0x0106}, {0x0043, 0x0302, 0x0108}, {0x0043, 0x0307, 0x010A}, + {0x0043, 0x030C, 0x010C}, {0x0043, 0x0327, 0x00C7}, {0x0043, 0x0341, 0x0106}, {0x0044, 0x0307, 0x1E0A}, + {0x0044, 0x030C, 0x010E}, {0x0044, 0x0323, 0x1E0C}, {0x0044, 0x0327, 0x1E10}, {0x0044, 0x032D, 0x1E12}, + {0x0044, 0x0331, 0x1E0E}, {0x0045, 0x0300, 0x00C8}, {0x0045, 0x0301, 0x00C9}, {0x0045, 0x0302, 0x00CA}, + {0x0045, 0x0303, 0x1EBC}, {0x0045, 0x0304, 0x0112}, {0x0045, 0x0306, 0x0114}, {0x0045, 0x0307, 0x0116}, + {0x0045, 0x0308, 0x00CB}, {0x0045, 0x0309, 0x1EBA}, {0x0045, 0x030C, 0x011A}, {0x0045, 0x030F, 0x0204}, + {0x0045, 0x0311, 0x0206}, {0x0045, 0x0323, 0x1EB8}, {0x0045, 0x0327, 0x0228}, {0x0045, 0x0328, 0x0118}, + {0x0045, 0x032D, 0x1E18}, {0x0045, 0x0330, 0x1E1A}, {0x0045, 0x0340, 0x00C8}, {0x0045, 0x0341, 0x00C9}, + {0x0046, 0x0307, 0x1E1E}, {0x0047, 0x0301, 0x01F4}, {0x0047, 0x0302, 0x011C}, {0x0047, 0x0304, 0x1E20}, + {0x0047, 0x0306, 0x011E}, {0x0047, 0x0307, 0x0120}, {0x0047, 0x030C, 0x01E6}, {0x0047, 0x0327, 0x0122}, + {0x0047, 0x0341, 0x01F4}, {0x0048, 0x0302, 0x0124}, {0x0048, 0x0307, 0x1E22}, {0x0048, 0x0308, 0x1E26}, + {0x0048, 0x030C, 0x021E}, {0x0048, 0x0323, 0x1E24}, {0x0048, 0x0327, 0x1E28}, {0x0048, 0x032E, 0x1E2A}, + {0x0049, 0x0300, 0x00CC}, {0x0049, 0x0301, 0x00CD}, {0x0049, 0x0302, 0x00CE}, {0x0049, 0x0303, 0x0128}, + {0x0049, 0x0304, 0x012A}, {0x0049, 0x0306, 0x012C}, {0x0049, 0x0307, 0x0130}, {0x0049, 0x0308, 0x00CF}, + {0x0049, 0x0309, 0x1EC8}, {0x0049, 0x030C, 0x01CF}, {0x0049, 0x030F, 0x0208}, {0x0049, 0x0311, 0x020A}, + {0x0049, 0x0323, 0x1ECA}, {0x0049, 0x0328, 0x012E}, {0x0049, 0x0330, 0x1E2C}, {0x0049, 0x0340, 0x00CC}, + {0x0049, 0x0341, 0x00CD}, {0x0049, 0x0344, 0x1E2E}, {0x004A, 0x0302, 0x0134}, {0x004B, 0x0301, 0x1E30}, + {0x004B, 0x030C, 0x01E8}, {0x004B, 0x0323, 0x1E32}, {0x004B, 0x0327, 0x0136}, {0x004B, 0x0331, 0x1E34}, + {0x004B, 0x0341, 0x1E30}, {0x004C, 0x0301, 0x0139}, {0x004C, 0x030C, 0x013D}, {0x004C, 0x0323, 0x1E36}, + {0x004C, 0x0327, 0x013B}, {0x004C, 0x032D, 0x1E3C}, {0x004C, 0x0331, 0x1E3A}, {0x004C, 0x0341, 0x0139}, + {0x004D, 0x0301, 0x1E3E}, {0x004D, 0x0307, 0x1E40}, {0x004D, 0x0323, 0x1E42}, {0x004D, 0x0341, 0x1E3E}, + {0x004E, 0x0300, 0x01F8}, {0x004E, 0x0301, 0x0143}, {0x004E, 0x0303, 0x00D1}, {0x004E, 0x0307, 0x1E44}, + {0x004E, 0x030C, 0x0147}, {0x004E, 0x0323, 0x1E46}, {0x004E, 0x0327, 0x0145}, {0x004E, 0x032D, 0x1E4A}, + {0x004E, 0x0331, 0x1E48}, {0x004E, 0x0340, 0x01F8}, {0x004E, 0x0341, 0x0143}, {0x004F, 0x0300, 0x00D2}, + {0x004F, 0x0301, 0x00D3}, {0x004F, 0x0302, 0x00D4}, {0x004F, 0x0303, 0x00D5}, {0x004F, 0x0304, 0x014C}, + {0x004F, 0x0306, 0x014E}, {0x004F, 0x0307, 0x022E}, {0x004F, 0x0308, 0x00D6}, {0x004F, 0x0309, 0x1ECE}, + {0x004F, 0x030B, 0x0150}, {0x004F, 0x030C, 0x01D1}, {0x004F, 0x030F, 0x020C}, {0x004F, 0x0311, 0x020E}, + {0x004F, 0x031B, 0x01A0}, {0x004F, 0x0323, 0x1ECC}, {0x004F, 0x0328, 0x01EA}, {0x004F, 0x0340, 0x00D2}, + {0x004F, 0x0341, 0x00D3}, {0x0050, 0x0301, 0x1E54}, {0x0050, 0x0307, 0x1E56}, {0x0050, 0x0341, 0x1E54}, + {0x0052, 0x0301, 0x0154}, {0x0052, 0x0307, 0x1E58}, {0x0052, 0x030C, 0x0158}, {0x0052, 0x030F, 0x0210}, + {0x0052, 0x0311, 0x0212}, {0x0052, 0x0323, 0x1E5A}, {0x0052, 0x0327, 0x0156}, {0x0052, 0x0331, 0x1E5E}, + {0x0052, 0x0341, 0x0154}, {0x0053, 0x0301, 0x015A}, {0x0053, 0x0302, 0x015C}, {0x0053, 0x0307, 0x1E60}, + {0x0053, 0x030C, 0x0160}, {0x0053, 0x0323, 0x1E62}, {0x0053, 0x0326, 0x0218}, {0x0053, 0x0327, 0x015E}, + {0x0053, 0x0341, 0x015A}, {0x0054, 0x0307, 0x1E6A}, {0x0054, 0x030C, 0x0164}, {0x0054, 0x0323, 0x1E6C}, + {0x0054, 0x0326, 0x021A}, {0x0054, 0x0327, 0x0162}, {0x0054, 0x032D, 0x1E70}, {0x0054, 0x0331, 0x1E6E}, + {0x0055, 0x0300, 0x00D9}, {0x0055, 0x0301, 0x00DA}, {0x0055, 0x0302, 0x00DB}, {0x0055, 0x0303, 0x0168}, + {0x0055, 0x0304, 0x016A}, {0x0055, 0x0306, 0x016C}, {0x0055, 0x0308, 0x00DC}, {0x0055, 0x0309, 0x1EE6}, + {0x0055, 0x030A, 0x016E}, {0x0055, 0x030B, 0x0170}, {0x0055, 0x030C, 0x01D3}, {0x0055, 0x030F, 0x0214}, + {0x0055, 0x0311, 0x0216}, {0x0055, 0x031B, 0x01AF}, {0x0055, 0x0323, 0x1EE4}, {0x0055, 0x0324, 0x1E72}, + {0x0055, 0x0328, 0x0172}, {0x0055, 0x032D, 0x1E76}, {0x0055, 0x0330, 0x1E74}, {0x0055, 0x0340, 0x00D9}, + {0x0055, 0x0341, 0x00DA}, {0x0055, 0x0344, 0x01D7}, {0x0056, 0x0303, 0x1E7C}, {0x0056, 0x0323, 0x1E7E}, + {0x0057, 0x0300, 0x1E80}, {0x0057, 0x0301, 0x1E82}, {0x0057, 0x0302, 0x0174}, {0x0057, 0x0307, 0x1E86}, + {0x0057, 0x0308, 0x1E84}, {0x0057, 0x0323, 0x1E88}, {0x0057, 0x0340, 0x1E80}, {0x0057, 0x0341, 0x1E82}, + {0x0058, 0x0307, 0x1E8A}, {0x0058, 0x0308, 0x1E8C}, {0x0059, 0x0300, 0x1EF2}, {0x0059, 0x0301, 0x00DD}, + {0x0059, 0x0302, 0x0176}, {0x0059, 0x0303, 0x1EF8}, {0x0059, 0x0304, 0x0232}, {0x0059, 0x0307, 0x1E8E}, + {0x0059, 0x0308, 0x0178}, {0x0059, 0x0309, 0x1EF6}, {0x0059, 0x0323, 0x1EF4}, {0x0059, 0x0340, 0x1EF2}, + {0x0059, 0x0341, 0x00DD}, {0x005A, 0x0301, 0x0179}, {0x005A, 0x0302, 0x1E90}, {0x005A, 0x0307, 0x017B}, + {0x005A, 0x030C, 0x017D}, {0x005A, 0x0323, 0x1E92}, {0x005A, 0x0331, 0x1E94}, {0x005A, 0x0341, 0x0179}, + {0x0061, 0x0300, 0x00E0}, {0x0061, 0x0301, 0x00E1}, {0x0061, 0x0302, 0x00E2}, {0x0061, 0x0303, 0x00E3}, + {0x0061, 0x0304, 0x0101}, {0x0061, 0x0306, 0x0103}, {0x0061, 0x0307, 0x0227}, {0x0061, 0x0308, 0x00E4}, + {0x0061, 0x0309, 0x1EA3}, {0x0061, 0x030A, 0x00E5}, {0x0061, 0x030C, 0x01CE}, {0x0061, 0x030F, 0x0201}, + {0x0061, 0x0311, 0x0203}, {0x0061, 0x0323, 0x1EA1}, {0x0061, 0x0325, 0x1E01}, {0x0061, 0x0328, 0x0105}, + {0x0061, 0x0340, 0x00E0}, {0x0061, 0x0341, 0x00E1}, {0x0062, 0x0307, 0x1E03}, {0x0062, 0x0323, 0x1E05}, + {0x0062, 0x0331, 0x1E07}, {0x0063, 0x0301, 0x0107}, {0x0063, 0x0302, 0x0109}, {0x0063, 0x0307, 0x010B}, + {0x0063, 0x030C, 0x010D}, {0x0063, 0x0327, 0x00E7}, {0x0063, 0x0341, 0x0107}, {0x0064, 0x0307, 0x1E0B}, + {0x0064, 0x030C, 0x010F}, {0x0064, 0x0323, 0x1E0D}, {0x0064, 0x0327, 0x1E11}, {0x0064, 0x032D, 0x1E13}, + {0x0064, 0x0331, 0x1E0F}, {0x0065, 0x0300, 0x00E8}, {0x0065, 0x0301, 0x00E9}, {0x0065, 0x0302, 0x00EA}, + {0x0065, 0x0303, 0x1EBD}, {0x0065, 0x0304, 0x0113}, {0x0065, 0x0306, 0x0115}, {0x0065, 0x0307, 0x0117}, + {0x0065, 0x0308, 0x00EB}, {0x0065, 0x0309, 0x1EBB}, {0x0065, 0x030C, 0x011B}, {0x0065, 0x030F, 0x0205}, + {0x0065, 0x0311, 0x0207}, {0x0065, 0x0323, 0x1EB9}, {0x0065, 0x0327, 0x0229}, {0x0065, 0x0328, 0x0119}, + {0x0065, 0x032D, 0x1E19}, {0x0065, 0x0330, 0x1E1B}, {0x0065, 0x0340, 0x00E8}, {0x0065, 0x0341, 0x00E9}, + {0x0066, 0x0307, 0x1E1F}, {0x0067, 0x0301, 0x01F5}, {0x0067, 0x0302, 0x011D}, {0x0067, 0x0304, 0x1E21}, + {0x0067, 0x0306, 0x011F}, {0x0067, 0x0307, 0x0121}, {0x0067, 0x030C, 0x01E7}, {0x0067, 0x0327, 0x0123}, + {0x0067, 0x0341, 0x01F5}, {0x0068, 0x0302, 0x0125}, {0x0068, 0x0307, 0x1E23}, {0x0068, 0x0308, 0x1E27}, + {0x0068, 0x030C, 0x021F}, {0x0068, 0x0323, 0x1E25}, {0x0068, 0x0327, 0x1E29}, {0x0068, 0x032E, 0x1E2B}, + {0x0068, 0x0331, 0x1E96}, {0x0069, 0x0300, 0x00EC}, {0x0069, 0x0301, 0x00ED}, {0x0069, 0x0302, 0x00EE}, + {0x0069, 0x0303, 0x0129}, {0x0069, 0x0304, 0x012B}, {0x0069, 0x0306, 0x012D}, {0x0069, 0x0308, 0x00EF}, + {0x0069, 0x0309, 0x1EC9}, {0x0069, 0x030C, 0x01D0}, {0x0069, 0x030F, 0x0209}, {0x0069, 0x0311, 0x020B}, + {0x0069, 0x0323, 0x1ECB}, {0x0069, 0x0328, 0x012F}, {0x0069, 0x0330, 0x1E2D}, {0x0069, 0x0340, 0x00EC}, + {0x0069, 0x0341, 0x00ED}, {0x0069, 0x0344, 0x1E2F}, {0x006A, 0x0302, 0x0135}, {0x006A, 0x030C, 0x01F0}, + {0x006B, 0x0301, 0x1E31}, {0x006B, 0x030C, 0x01E9}, {0x006B, 0x0323, 0x1E33}, {0x006B, 0x0327, 0x0137}, + {0x006B, 0x0331, 0x1E35}, {0x006B, 0x0341, 0x1E31}, {0x006C, 0x0301, 0x013A}, {0x006C, 0x030C, 0x013E}, + {0x006C, 0x0323, 0x1E37}, {0x006C, 0x0327, 0x013C}, {0x006C, 0x032D, 0x1E3D}, {0x006C, 0x0331, 0x1E3B}, + {0x006C, 0x0341, 0x013A}, {0x006D, 0x0301, 0x1E3F}, {0x006D, 0x0307, 0x1E41}, {0x006D, 0x0323, 0x1E43}, + {0x006D, 0x0341, 0x1E3F}, {0x006E, 0x0300, 0x01F9}, {0x006E, 0x0301, 0x0144}, {0x006E, 0x0303, 0x00F1}, + {0x006E, 0x0307, 0x1E45}, {0x006E, 0x030C, 0x0148}, {0x006E, 0x0323, 0x1E47}, {0x006E, 0x0327, 0x0146}, + {0x006E, 0x032D, 0x1E4B}, {0x006E, 0x0331, 0x1E49}, {0x006E, 0x0340, 0x01F9}, {0x006E, 0x0341, 0x0144}, + {0x006F, 0x0300, 0x00F2}, {0x006F, 0x0301, 0x00F3}, {0x006F, 0x0302, 0x00F4}, {0x006F, 0x0303, 0x00F5}, + {0x006F, 0x0304, 0x014D}, {0x006F, 0x0306, 0x014F}, {0x006F, 0x0307, 0x022F}, {0x006F, 0x0308, 0x00F6}, + {0x006F, 0x0309, 0x1ECF}, {0x006F, 0x030B, 0x0151}, {0x006F, 0x030C, 0x01D2}, {0x006F, 0x030F, 0x020D}, + {0x006F, 0x0311, 0x020F}, {0x006F, 0x031B, 0x01A1}, {0x006F, 0x0323, 0x1ECD}, {0x006F, 0x0328, 0x01EB}, + {0x006F, 0x0340, 0x00F2}, {0x006F, 0x0341, 0x00F3}, {0x0070, 0x0301, 0x1E55}, {0x0070, 0x0307, 0x1E57}, + {0x0070, 0x0341, 0x1E55}, {0x0072, 0x0301, 0x0155}, {0x0072, 0x0307, 0x1E59}, {0x0072, 0x030C, 0x0159}, + {0x0072, 0x030F, 0x0211}, {0x0072, 0x0311, 0x0213}, {0x0072, 0x0323, 0x1E5B}, {0x0072, 0x0327, 0x0157}, + {0x0072, 0x0331, 0x1E5F}, {0x0072, 0x0341, 0x0155}, {0x0073, 0x0301, 0x015B}, {0x0073, 0x0302, 0x015D}, + {0x0073, 0x0307, 0x1E61}, {0x0073, 0x030C, 0x0161}, {0x0073, 0x0323, 0x1E63}, {0x0073, 0x0326, 0x0219}, + {0x0073, 0x0327, 0x015F}, {0x0073, 0x0341, 0x015B}, {0x0074, 0x0307, 0x1E6B}, {0x0074, 0x0308, 0x1E97}, + {0x0074, 0x030C, 0x0165}, {0x0074, 0x0323, 0x1E6D}, {0x0074, 0x0326, 0x021B}, {0x0074, 0x0327, 0x0163}, + {0x0074, 0x032D, 0x1E71}, {0x0074, 0x0331, 0x1E6F}, {0x0075, 0x0300, 0x00F9}, {0x0075, 0x0301, 0x00FA}, + {0x0075, 0x0302, 0x00FB}, {0x0075, 0x0303, 0x0169}, {0x0075, 0x0304, 0x016B}, {0x0075, 0x0306, 0x016D}, + {0x0075, 0x0308, 0x00FC}, {0x0075, 0x0309, 0x1EE7}, {0x0075, 0x030A, 0x016F}, {0x0075, 0x030B, 0x0171}, + {0x0075, 0x030C, 0x01D4}, {0x0075, 0x030F, 0x0215}, {0x0075, 0x0311, 0x0217}, {0x0075, 0x031B, 0x01B0}, + {0x0075, 0x0323, 0x1EE5}, {0x0075, 0x0324, 0x1E73}, {0x0075, 0x0328, 0x0173}, {0x0075, 0x032D, 0x1E77}, + {0x0075, 0x0330, 0x1E75}, {0x0075, 0x0340, 0x00F9}, {0x0075, 0x0341, 0x00FA}, {0x0075, 0x0344, 0x01D8}, + {0x0076, 0x0303, 0x1E7D}, {0x0076, 0x0323, 0x1E7F}, {0x0077, 0x0300, 0x1E81}, {0x0077, 0x0301, 0x1E83}, + {0x0077, 0x0302, 0x0175}, {0x0077, 0x0307, 0x1E87}, {0x0077, 0x0308, 0x1E85}, {0x0077, 0x030A, 0x1E98}, + {0x0077, 0x0323, 0x1E89}, {0x0077, 0x0340, 0x1E81}, {0x0077, 0x0341, 0x1E83}, {0x0078, 0x0307, 0x1E8B}, + {0x0078, 0x0308, 0x1E8D}, {0x0079, 0x0300, 0x1EF3}, {0x0079, 0x0301, 0x00FD}, {0x0079, 0x0302, 0x0177}, + {0x0079, 0x0303, 0x1EF9}, {0x0079, 0x0304, 0x0233}, {0x0079, 0x0307, 0x1E8F}, {0x0079, 0x0308, 0x00FF}, + {0x0079, 0x0309, 0x1EF7}, {0x0079, 0x030A, 0x1E99}, {0x0079, 0x0323, 0x1EF5}, {0x0079, 0x0340, 0x1EF3}, + {0x0079, 0x0341, 0x00FD}, {0x007A, 0x0301, 0x017A}, {0x007A, 0x0302, 0x1E91}, {0x007A, 0x0307, 0x017C}, + {0x007A, 0x030C, 0x017E}, {0x007A, 0x0323, 0x1E93}, {0x007A, 0x0331, 0x1E95}, {0x007A, 0x0341, 0x017A}, + {0x00A8, 0x0300, 0x1FED}, {0x00A8, 0x0301, 0x0385}, {0x00A8, 0x0340, 0x1FED}, {0x00A8, 0x0341, 0x0385}, + {0x00A8, 0x0342, 0x1FC1}, {0x00C2, 0x0300, 0x1EA6}, {0x00C2, 0x0301, 0x1EA4}, {0x00C2, 0x0303, 0x1EAA}, + {0x00C2, 0x0309, 0x1EA8}, {0x00C2, 0x0323, 0x1EAC}, {0x00C2, 0x0340, 0x1EA6}, {0x00C2, 0x0341, 0x1EA4}, + {0x00C4, 0x0304, 0x01DE}, {0x00C5, 0x0301, 0x01FA}, {0x00C5, 0x0341, 0x01FA}, {0x00C6, 0x0301, 0x01FC}, + {0x00C6, 0x0304, 0x01E2}, {0x00C6, 0x0341, 0x01FC}, {0x00C7, 0x0301, 0x1E08}, {0x00C7, 0x0341, 0x1E08}, + {0x00CA, 0x0300, 0x1EC0}, {0x00CA, 0x0301, 0x1EBE}, {0x00CA, 0x0303, 0x1EC4}, {0x00CA, 0x0309, 0x1EC2}, + {0x00CA, 0x0323, 0x1EC6}, {0x00CA, 0x0340, 0x1EC0}, {0x00CA, 0x0341, 0x1EBE}, {0x00CF, 0x0301, 0x1E2E}, + {0x00CF, 0x0341, 0x1E2E}, {0x00D2, 0x031B, 0x1EDC}, {0x00D3, 0x031B, 0x1EDA}, {0x00D4, 0x0300, 0x1ED2}, + {0x00D4, 0x0301, 0x1ED0}, {0x00D4, 0x0303, 0x1ED6}, {0x00D4, 0x0309, 0x1ED4}, {0x00D4, 0x0323, 0x1ED8}, + {0x00D4, 0x0340, 0x1ED2}, {0x00D4, 0x0341, 0x1ED0}, {0x00D5, 0x0301, 0x1E4C}, {0x00D5, 0x0304, 0x022C}, + {0x00D5, 0x0308, 0x1E4E}, {0x00D5, 0x031B, 0x1EE0}, {0x00D5, 0x0341, 0x1E4C}, {0x00D6, 0x0304, 0x022A}, + {0x00D8, 0x0301, 0x01FE}, {0x00D8, 0x0341, 0x01FE}, {0x00D9, 0x031B, 0x1EEA}, {0x00DA, 0x031B, 0x1EE8}, + {0x00DC, 0x0300, 0x01DB}, {0x00DC, 0x0301, 0x01D7}, {0x00DC, 0x0304, 0x01D5}, {0x00DC, 0x030C, 0x01D9}, + {0x00DC, 0x0340, 0x01DB}, {0x00DC, 0x0341, 0x01D7}, {0x00E2, 0x0300, 0x1EA7}, {0x00E2, 0x0301, 0x1EA5}, + {0x00E2, 0x0303, 0x1EAB}, {0x00E2, 0x0309, 0x1EA9}, {0x00E2, 0x0323, 0x1EAD}, {0x00E2, 0x0340, 0x1EA7}, + {0x00E2, 0x0341, 0x1EA5}, {0x00E4, 0x0304, 0x01DF}, {0x00E5, 0x0301, 0x01FB}, {0x00E5, 0x0341, 0x01FB}, + {0x00E6, 0x0301, 0x01FD}, {0x00E6, 0x0304, 0x01E3}, {0x00E6, 0x0341, 0x01FD}, {0x00E7, 0x0301, 0x1E09}, + {0x00E7, 0x0341, 0x1E09}, {0x00EA, 0x0300, 0x1EC1}, {0x00EA, 0x0301, 0x1EBF}, {0x00EA, 0x0303, 0x1EC5}, + {0x00EA, 0x0309, 0x1EC3}, {0x00EA, 0x0323, 0x1EC7}, {0x00EA, 0x0340, 0x1EC1}, {0x00EA, 0x0341, 0x1EBF}, + {0x00EF, 0x0301, 0x1E2F}, {0x00EF, 0x0341, 0x1E2F}, {0x00F2, 0x031B, 0x1EDD}, {0x00F3, 0x031B, 0x1EDB}, + {0x00F4, 0x0300, 0x1ED3}, {0x00F4, 0x0301, 0x1ED1}, {0x00F4, 0x0303, 0x1ED7}, {0x00F4, 0x0309, 0x1ED5}, + {0x00F4, 0x0323, 0x1ED9}, {0x00F4, 0x0340, 0x1ED3}, {0x00F4, 0x0341, 0x1ED1}, {0x00F5, 0x0301, 0x1E4D}, + {0x00F5, 0x0304, 0x022D}, {0x00F5, 0x0308, 0x1E4F}, {0x00F5, 0x031B, 0x1EE1}, {0x00F5, 0x0341, 0x1E4D}, + {0x00F6, 0x0304, 0x022B}, {0x00F8, 0x0301, 0x01FF}, {0x00F8, 0x0341, 0x01FF}, {0x00F9, 0x031B, 0x1EEB}, + {0x00FA, 0x031B, 0x1EE9}, {0x00FC, 0x0300, 0x01DC}, {0x00FC, 0x0301, 0x01D8}, {0x00FC, 0x0304, 0x01D6}, + {0x00FC, 0x030C, 0x01DA}, {0x00FC, 0x0340, 0x01DC}, {0x00FC, 0x0341, 0x01D8}, {0x0102, 0x0300, 0x1EB0}, + {0x0102, 0x0301, 0x1EAE}, {0x0102, 0x0303, 0x1EB4}, {0x0102, 0x0309, 0x1EB2}, {0x0102, 0x0323, 0x1EB6}, + {0x0102, 0x0340, 0x1EB0}, {0x0102, 0x0341, 0x1EAE}, {0x0103, 0x0300, 0x1EB1}, {0x0103, 0x0301, 0x1EAF}, + {0x0103, 0x0303, 0x1EB5}, {0x0103, 0x0309, 0x1EB3}, {0x0103, 0x0323, 0x1EB7}, {0x0103, 0x0340, 0x1EB1}, + {0x0103, 0x0341, 0x1EAF}, {0x0106, 0x0327, 0x1E08}, {0x0107, 0x0327, 0x1E09}, {0x0112, 0x0300, 0x1E14}, + {0x0112, 0x0301, 0x1E16}, {0x0112, 0x0340, 0x1E14}, {0x0112, 0x0341, 0x1E16}, {0x0113, 0x0300, 0x1E15}, + {0x0113, 0x0301, 0x1E17}, {0x0113, 0x0340, 0x1E15}, {0x0113, 0x0341, 0x1E17}, {0x0114, 0x0327, 0x1E1C}, + {0x0115, 0x0327, 0x1E1D}, {0x014C, 0x0300, 0x1E50}, {0x014C, 0x0301, 0x1E52}, {0x014C, 0x0328, 0x01EC}, + {0x014C, 0x0340, 0x1E50}, {0x014C, 0x0341, 0x1E52}, {0x014D, 0x0300, 0x1E51}, {0x014D, 0x0301, 0x1E53}, + {0x014D, 0x0328, 0x01ED}, {0x014D, 0x0340, 0x1E51}, {0x014D, 0x0341, 0x1E53}, {0x015A, 0x0307, 0x1E64}, + {0x015B, 0x0307, 0x1E65}, {0x0160, 0x0307, 0x1E66}, {0x0161, 0x0307, 0x1E67}, {0x0168, 0x0301, 0x1E78}, + {0x0168, 0x031B, 0x1EEE}, {0x0168, 0x0341, 0x1E78}, {0x0169, 0x0301, 0x1E79}, {0x0169, 0x031B, 0x1EEF}, + {0x0169, 0x0341, 0x1E79}, {0x016A, 0x0308, 0x1E7A}, {0x016B, 0x0308, 0x1E7B}, {0x017F, 0x0307, 0x1E9B}, + {0x01A0, 0x0300, 0x1EDC}, {0x01A0, 0x0301, 0x1EDA}, {0x01A0, 0x0303, 0x1EE0}, {0x01A0, 0x0309, 0x1EDE}, + {0x01A0, 0x0323, 0x1EE2}, {0x01A0, 0x0340, 0x1EDC}, {0x01A0, 0x0341, 0x1EDA}, {0x01A1, 0x0300, 0x1EDD}, + {0x01A1, 0x0301, 0x1EDB}, {0x01A1, 0x0303, 0x1EE1}, {0x01A1, 0x0309, 0x1EDF}, {0x01A1, 0x0323, 0x1EE3}, + {0x01A1, 0x0340, 0x1EDD}, {0x01A1, 0x0341, 0x1EDB}, {0x01AF, 0x0300, 0x1EEA}, {0x01AF, 0x0301, 0x1EE8}, + {0x01AF, 0x0303, 0x1EEE}, {0x01AF, 0x0309, 0x1EEC}, {0x01AF, 0x0323, 0x1EF0}, {0x01AF, 0x0340, 0x1EEA}, + {0x01AF, 0x0341, 0x1EE8}, {0x01B0, 0x0300, 0x1EEB}, {0x01B0, 0x0301, 0x1EE9}, {0x01B0, 0x0303, 0x1EEF}, + {0x01B0, 0x0309, 0x1EED}, {0x01B0, 0x0323, 0x1EF1}, {0x01B0, 0x0340, 0x1EEB}, {0x01B0, 0x0341, 0x1EE9}, + {0x01B7, 0x030C, 0x01EE}, {0x01EA, 0x0304, 0x01EC}, {0x01EB, 0x0304, 0x01ED}, {0x0226, 0x0304, 0x01E0}, + {0x0227, 0x0304, 0x01E1}, {0x0228, 0x0306, 0x1E1C}, {0x0229, 0x0306, 0x1E1D}, {0x022E, 0x0304, 0x0230}, + {0x022F, 0x0304, 0x0231}, {0x0292, 0x030C, 0x01EF}, {0x0391, 0x0300, 0x1FBA}, {0x0391, 0x0301, 0x0386}, + {0x0391, 0x0304, 0x1FB9}, {0x0391, 0x0306, 0x1FB8}, {0x0391, 0x0313, 0x1F08}, {0x0391, 0x0314, 0x1F09}, + {0x0391, 0x0340, 0x1FBA}, {0x0391, 0x0341, 0x0386}, {0x0391, 0x0343, 0x1F08}, {0x0391, 0x0345, 0x1FBC}, + {0x0395, 0x0300, 0x1FC8}, {0x0395, 0x0301, 0x0388}, {0x0395, 0x0313, 0x1F18}, {0x0395, 0x0314, 0x1F19}, + {0x0395, 0x0340, 0x1FC8}, {0x0395, 0x0341, 0x0388}, {0x0395, 0x0343, 0x1F18}, {0x0397, 0x0300, 0x1FCA}, + {0x0397, 0x0301, 0x0389}, {0x0397, 0x0313, 0x1F28}, {0x0397, 0x0314, 0x1F29}, {0x0397, 0x0340, 0x1FCA}, + {0x0397, 0x0341, 0x0389}, {0x0397, 0x0343, 0x1F28}, {0x0397, 0x0345, 0x1FCC}, {0x0399, 0x0300, 0x1FDA}, + {0x0399, 0x0301, 0x038A}, {0x0399, 0x0304, 0x1FD9}, {0x0399, 0x0306, 0x1FD8}, {0x0399, 0x0308, 0x03AA}, + {0x0399, 0x0313, 0x1F38}, {0x0399, 0x0314, 0x1F39}, {0x0399, 0x0340, 0x1FDA}, {0x0399, 0x0341, 0x038A}, + {0x0399, 0x0343, 0x1F38}, {0x039F, 0x0300, 0x1FF8}, {0x039F, 0x0301, 0x038C}, {0x039F, 0x0313, 0x1F48}, + {0x039F, 0x0314, 0x1F49}, {0x039F, 0x0340, 0x1FF8}, {0x039F, 0x0341, 0x038C}, {0x039F, 0x0343, 0x1F48}, + {0x03A1, 0x0314, 0x1FEC}, {0x03A5, 0x0300, 0x1FEA}, {0x03A5, 0x0301, 0x038E}, {0x03A5, 0x0304, 0x1FE9}, + {0x03A5, 0x0306, 0x1FE8}, {0x03A5, 0x0308, 0x03AB}, {0x03A5, 0x0314, 0x1F59}, {0x03A5, 0x0340, 0x1FEA}, + {0x03A5, 0x0341, 0x038E}, {0x03A9, 0x0300, 0x1FFA}, {0x03A9, 0x0301, 0x038F}, {0x03A9, 0x0313, 0x1F68}, + {0x03A9, 0x0314, 0x1F69}, {0x03A9, 0x0340, 0x1FFA}, {0x03A9, 0x0341, 0x038F}, {0x03A9, 0x0343, 0x1F68}, + {0x03A9, 0x0345, 0x1FFC}, {0x03AC, 0x0345, 0x1FB4}, {0x03AE, 0x0345, 0x1FC4}, {0x03B1, 0x0300, 0x1F70}, + {0x03B1, 0x0301, 0x03AC}, {0x03B1, 0x0304, 0x1FB1}, {0x03B1, 0x0306, 0x1FB0}, {0x03B1, 0x0313, 0x1F00}, + {0x03B1, 0x0314, 0x1F01}, {0x03B1, 0x0340, 0x1F70}, {0x03B1, 0x0341, 0x03AC}, {0x03B1, 0x0342, 0x1FB6}, + {0x03B1, 0x0343, 0x1F00}, {0x03B1, 0x0345, 0x1FB3}, {0x03B5, 0x0300, 0x1F72}, {0x03B5, 0x0301, 0x03AD}, + {0x03B5, 0x0313, 0x1F10}, {0x03B5, 0x0314, 0x1F11}, {0x03B5, 0x0340, 0x1F72}, {0x03B5, 0x0341, 0x03AD}, + {0x03B5, 0x0343, 0x1F10}, {0x03B7, 0x0300, 0x1F74}, {0x03B7, 0x0301, 0x03AE}, {0x03B7, 0x0313, 0x1F20}, + {0x03B7, 0x0314, 0x1F21}, {0x03B7, 0x0340, 0x1F74}, {0x03B7, 0x0341, 0x03AE}, {0x03B7, 0x0342, 0x1FC6}, + {0x03B7, 0x0343, 0x1F20}, {0x03B7, 0x0345, 0x1FC3}, {0x03B9, 0x0300, 0x1F76}, {0x03B9, 0x0301, 0x03AF}, + {0x03B9, 0x0304, 0x1FD1}, {0x03B9, 0x0306, 0x1FD0}, {0x03B9, 0x0308, 0x03CA}, {0x03B9, 0x0313, 0x1F30}, + {0x03B9, 0x0314, 0x1F31}, {0x03B9, 0x0340, 0x1F76}, {0x03B9, 0x0341, 0x03AF}, {0x03B9, 0x0342, 0x1FD6}, + {0x03B9, 0x0343, 0x1F30}, {0x03B9, 0x0344, 0x0390}, {0x03BF, 0x0300, 0x1F78}, {0x03BF, 0x0301, 0x03CC}, + {0x03BF, 0x0313, 0x1F40}, {0x03BF, 0x0314, 0x1F41}, {0x03BF, 0x0340, 0x1F78}, {0x03BF, 0x0341, 0x03CC}, + {0x03BF, 0x0343, 0x1F40}, {0x03C1, 0x0313, 0x1FE4}, {0x03C1, 0x0314, 0x1FE5}, {0x03C1, 0x0343, 0x1FE4}, + {0x03C5, 0x0300, 0x1F7A}, {0x03C5, 0x0301, 0x03CD}, {0x03C5, 0x0304, 0x1FE1}, {0x03C5, 0x0306, 0x1FE0}, + {0x03C5, 0x0308, 0x03CB}, {0x03C5, 0x0313, 0x1F50}, {0x03C5, 0x0314, 0x1F51}, {0x03C5, 0x0340, 0x1F7A}, + {0x03C5, 0x0341, 0x03CD}, {0x03C5, 0x0342, 0x1FE6}, {0x03C5, 0x0343, 0x1F50}, {0x03C5, 0x0344, 0x03B0}, + {0x03C9, 0x0300, 0x1F7C}, {0x03C9, 0x0301, 0x03CE}, {0x03C9, 0x0313, 0x1F60}, {0x03C9, 0x0314, 0x1F61}, + {0x03C9, 0x0340, 0x1F7C}, {0x03C9, 0x0341, 0x03CE}, {0x03C9, 0x0342, 0x1FF6}, {0x03C9, 0x0343, 0x1F60}, + {0x03C9, 0x0345, 0x1FF3}, {0x03CA, 0x0300, 0x1FD2}, {0x03CA, 0x0301, 0x0390}, {0x03CA, 0x0340, 0x1FD2}, + {0x03CA, 0x0341, 0x0390}, {0x03CA, 0x0342, 0x1FD7}, {0x03CB, 0x0300, 0x1FE2}, {0x03CB, 0x0301, 0x03B0}, + {0x03CB, 0x0340, 0x1FE2}, {0x03CB, 0x0341, 0x03B0}, {0x03CB, 0x0342, 0x1FE7}, {0x03CE, 0x0345, 0x1FF4}, + {0x03D2, 0x0301, 0x03D3}, {0x03D2, 0x0308, 0x03D4}, {0x03D2, 0x0341, 0x03D3}, {0x0406, 0x0308, 0x0407}, + {0x0410, 0x0306, 0x04D0}, {0x0410, 0x0308, 0x04D2}, {0x0413, 0x0301, 0x0403}, {0x0413, 0x0341, 0x0403}, + {0x0415, 0x0300, 0x0400}, {0x0415, 0x0306, 0x04D6}, {0x0415, 0x0308, 0x0401}, {0x0415, 0x0340, 0x0400}, + {0x0416, 0x0306, 0x04C1}, {0x0416, 0x0308, 0x04DC}, {0x0417, 0x0308, 0x04DE}, {0x0418, 0x0300, 0x040D}, + {0x0418, 0x0304, 0x04E2}, {0x0418, 0x0306, 0x0419}, {0x0418, 0x0308, 0x04E4}, {0x0418, 0x0340, 0x040D}, + {0x041A, 0x0301, 0x040C}, {0x041A, 0x0341, 0x040C}, {0x041E, 0x0308, 0x04E6}, {0x0423, 0x0304, 0x04EE}, + {0x0423, 0x0306, 0x040E}, {0x0423, 0x0308, 0x04F0}, {0x0423, 0x030B, 0x04F2}, {0x0427, 0x0308, 0x04F4}, + {0x042B, 0x0308, 0x04F8}, {0x042D, 0x0308, 0x04EC}, {0x0430, 0x0306, 0x04D1}, {0x0430, 0x0308, 0x04D3}, + {0x0433, 0x0301, 0x0453}, {0x0433, 0x0341, 0x0453}, {0x0435, 0x0300, 0x0450}, {0x0435, 0x0306, 0x04D7}, + {0x0435, 0x0308, 0x0451}, {0x0435, 0x0340, 0x0450}, {0x0436, 0x0306, 0x04C2}, {0x0436, 0x0308, 0x04DD}, + {0x0437, 0x0308, 0x04DF}, {0x0438, 0x0300, 0x045D}, {0x0438, 0x0304, 0x04E3}, {0x0438, 0x0306, 0x0439}, + {0x0438, 0x0308, 0x04E5}, {0x0438, 0x0340, 0x045D}, {0x043A, 0x0301, 0x045C}, {0x043A, 0x0341, 0x045C}, + {0x043E, 0x0308, 0x04E7}, {0x0443, 0x0304, 0x04EF}, {0x0443, 0x0306, 0x045E}, {0x0443, 0x0308, 0x04F1}, + {0x0443, 0x030B, 0x04F3}, {0x0447, 0x0308, 0x04F5}, {0x044B, 0x0308, 0x04F9}, {0x044D, 0x0308, 0x04ED}, + {0x0456, 0x0308, 0x0457}, {0x0474, 0x030F, 0x0476}, {0x0475, 0x030F, 0x0477}, {0x04D8, 0x0308, 0x04DA}, + {0x04D9, 0x0308, 0x04DB}, {0x04E8, 0x0308, 0x04EA}, {0x04E9, 0x0308, 0x04EB}, {0x1E36, 0x0304, 0x1E38}, + {0x1E37, 0x0304, 0x1E39}, {0x1E5A, 0x0304, 0x1E5C}, {0x1E5B, 0x0304, 0x1E5D}, {0x1E60, 0x0323, 0x1E68}, + {0x1E61, 0x0323, 0x1E69}, {0x1E62, 0x0307, 0x1E68}, {0x1E63, 0x0307, 0x1E69}, {0x1EA0, 0x0302, 0x1EAC}, + {0x1EA0, 0x0306, 0x1EB6}, {0x1EA1, 0x0302, 0x1EAD}, {0x1EA1, 0x0306, 0x1EB7}, {0x1EB8, 0x0302, 0x1EC6}, + {0x1EB9, 0x0302, 0x1EC7}, {0x1ECC, 0x0302, 0x1ED8}, {0x1ECC, 0x031B, 0x1EE2}, {0x1ECD, 0x0302, 0x1ED9}, + {0x1ECD, 0x031B, 0x1EE3}, {0x1ECE, 0x031B, 0x1EDE}, {0x1ECF, 0x031B, 0x1EDF}, {0x1EE4, 0x031B, 0x1EF0}, + {0x1EE5, 0x031B, 0x1EF1}, {0x1EE6, 0x031B, 0x1EEC}, {0x1EE7, 0x031B, 0x1EED}, +}; +static constexpr int kUtf8ComposeTableSize = sizeof(kUtf8ComposeTable) / sizeof(kUtf8ComposeTable[0]); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 31ce26bf..17c46643 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -42,3 +42,4 @@ add_subdirectory(streaming_json_parser) add_subdirectory(release_json_parser) add_subdirectory(differential_rounding) add_subdirectory(hyphenation_eval) +add_subdirectory(utf8_compose) diff --git a/test/utf8_compose/CMakeLists.txt b/test/utf8_compose/CMakeLists.txt new file mode 100644 index 00000000..b567d063 --- /dev/null +++ b/test/utf8_compose/CMakeLists.txt @@ -0,0 +1,15 @@ +add_executable(Utf8ComposeTest + Utf8ComposeTest.cpp + ${REPO_ROOT}/lib/Utf8/Utf8.cpp +) + +target_include_directories(Utf8ComposeTest PRIVATE + ${REPO_ROOT}/lib/Utf8 +) + +target_link_libraries(Utf8ComposeTest PRIVATE + crosspoint_test_common + GTest::gtest_main +) + +gtest_discover_tests(Utf8ComposeTest) diff --git a/test/utf8_compose/Utf8ComposeTest.cpp b/test/utf8_compose/Utf8ComposeTest.cpp new file mode 100644 index 00000000..30f142b4 --- /dev/null +++ b/test/utf8_compose/Utf8ComposeTest.cpp @@ -0,0 +1,54 @@ +#include + +#include + +#include "Utf8.h" + +namespace { + +// Helpers to build NFD / expected byte sequences explicitly so the test does not +// depend on the encoding of this source file. +const std::string kCombGrave = "\xCC\x80"; // U+0300 COMBINING GRAVE ACCENT +const std::string kCombAcute = "\xCC\x81"; // U+0301 COMBINING ACUTE ACCENT +const std::string kCombCirc = "\xCC\x82"; // U+0302 COMBINING CIRCUMFLEX ACCENT +const std::string kCombDotBelow = "\xCC\xA3"; // U+0323 COMBINING DOT BELOW + +} // namespace + +// ASCII and already-precomposed (NFC) text must pass through untouched (fast path). +TEST(Utf8ComposeNfc, PassesThroughAsciiAndNfc) { + EXPECT_EQ(utf8ComposeNfc(""), ""); + EXPECT_EQ(utf8ComposeNfc("hello world"), "hello world"); + EXPECT_EQ(utf8ComposeNfc("caf\xC3\xA9"), "caf\xC3\xA9"); // é already U+00E9 +} + +// Single combining mark composes onto its base letter. +TEST(Utf8ComposeNfc, ComposesSingleMark) { + EXPECT_EQ(utf8ComposeNfc("e" + kCombAcute), "\xC3\xA9"); // e + ́ -> é (U+00E9) + EXPECT_EQ(utf8ComposeNfc("a" + kCombGrave), "\xC3\xA0"); // a + ̀ -> à (U+00E0) +} + +// Vietnamese letters carry two stacked marks; composition must accumulate them +// onto the intermediate precomposed character (this is the crux of the feature). +TEST(Utf8ComposeNfc, ComposesStackedVietnameseMarks) { + // a + circumflex + acute -> ấ (U+1EA5) + EXPECT_EQ(utf8ComposeNfc("a" + kCombCirc + kCombAcute), "\xE1\xBA\xA5"); + // a + dot-below + circumflex (canonical order) -> ậ (U+1EAD) + EXPECT_EQ(utf8ComposeNfc("a" + kCombDotBelow + kCombCirc), "\xE1\xBA\xAD"); +} + +// A combining mark with no composition for its base is left unchanged, and the +// base is preserved. +TEST(Utf8ComposeNfc, LeavesUncomposableMarksIntact) { + const std::string in = "q" + kCombAcute; // no precomposed "q with acute" + EXPECT_EQ(utf8ComposeNfc(in), in); +} + +// A leading combining mark (no preceding base) is emitted unchanged. +TEST(Utf8ComposeNfc, HandlesLeadingMark) { EXPECT_EQ(utf8ComposeNfc(kCombAcute), kCombAcute); } + +// Marks embedded in a longer word compose while surrounding text is preserved. +TEST(Utf8ComposeNfc, ComposesWithinWord) { + // "Ti" + e+circ+acute + "ng" -> "Tiếng" + EXPECT_EQ(utf8ComposeNfc("Ti" + std::string("e") + kCombCirc + kCombAcute + "ng"), "Ti\xE1\xBA\xBFng"); +}