eric/Crosspoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(epub): NFC-normalize EPUB text so NFD diacritics render correctly (#2277)

Browse Source 
Co-authored-by: Uri Tauber <uritaube@gmail.com>
Co-authored-by: Julia <julia@uxj.io>
This commit is contained in:
Hoang Manh Linh
2026-06-18 13:00:01 +03:00
committed by GitHub
co-authored by Uri Tauber Julia
parent 7d639cf880
commit 16b0853654
10 changed files with 392 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Epub/Epub.cpp
+4 -2
View File
@@ -5,6 +5,7 @@
#include <JpegToBmpConverter.h>
#include <Logging.h>
#include <PngToBmpConverter.h>
#include <Utf8.h>
#include <ZipFile.h>
#include "Epub/parsers/ContainerParser.h"
@@ -73,8 +74,9 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, const
return false;
}
// Grab data from opfParser into epub
bookMetadata.title = opfParser.title;
// Grab data from opfParser into epub. Normalize titles to NFC so NFD (combining
// mark) text renders correctly — the device fonts have no mark positioning.
bookMetadata.title = utf8ComposeNfc(opfParser.title);
bookMetadata.author = opfParser.author;
bookMetadata.language = opfParser.language;
bookMetadata.coverItemHref = opfParser.coverItemHref;
lib/Epub/Epub/BookMetadataCache.cpp
+5 -2
View File
@@ -2,6 +2,7 @@
#include <Logging.h>
#include <Serialization.h>
#include <Utf8.h>
#include <ZipFile.h>
#include <deque>
@@ -9,7 +10,7 @@
#include "FsHelpers.h"
namespace {
constexpr uint8_t BOOK_CACHE_VERSION = 7;
constexpr uint8_t BOOK_CACHE_VERSION = 8; // v8: TOC/book titles stored NFC-composed
constexpr char bookBinFile[] = "/book.bin";
constexpr char tmpSpineBinFile[] = "/spine.bin.tmp";
constexpr char tmpTocBinFile[] = "/toc.bin.tmp";
@@ -364,7 +365,9 @@ void BookMetadataCache::createTocEntry(const std::string& title, const std::stri
}
}
const TocEntry entry(title, href, anchor, level, spineIndex);
// Compose the title to NFC at index time so the cache stores precomposed glyphs;
// device fonts have no combining-mark positioning, so NFD titles render broken.
const TocEntry entry(utf8ComposeNfc(title), href, anchor, level, spineIndex);
writeTocEntry(tocFile, entry);
tocCount++;
}
lib/Epub/Epub/ParsedText.cpp
+8
View File
@@ -254,6 +254,14 @@ void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle,
const bool attachToPrevious) {
if (word.empty()) return;
// The device fonts carry no combining-mark positioning, so EPUB text stored in NFD
// (a base letter followed by separate combining accents -- common for Vietnamese,
// and used for many EPUB <h1> chapter headings) renders with the marks detached or
// misplaced. Compose to NFC here, the single funnel every word passes through, so a
// precomposed glyph is used instead. This runs once per word at layout time (the
// result is cached in the section file) and is a cheap no-op for mark-free text.
word = utf8ComposeNfc(word);
EpdFontFamily::Style baseStyle = fontStyle;
if (underline) {
baseStyle = static_cast<EpdFontFamily::Style>(baseStyle | EpdFontFamily::UNDERLINE);
lib/Epub/Epub/Section.cpp
+2 -1
View File
@@ -10,7 +10,8 @@
#include "parsers/ChapterHtmlSlimParser.h"
namespace {
constexpr uint8_t SECTION_FILE_VERSION = 26;
// v27: words NFC-composed at layout time; bump invalidates NFD section caches.
constexpr uint8_t SECTION_FILE_VERSION = 27;
constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(uint8_t) +
sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) + sizeof(bool) +
sizeof(uint8_t) + sizeof(bool) + sizeof(uint32_t) + sizeof(uint32_t) +
lib/Utf8/Utf8.cpp
+68
View File
@@ -1,5 +1,73 @@
#include "Utf8.h"
#include "Utf8ComposeTable.h"
namespace {
// Look up the canonical composition of (base + combining mark), or 0 if none.
uint32_t utf8ComposePair(const uint32_t base, const uint32_t mark) {
if (base > 0xFFFF || mark > 0xFFFF) return 0;
int lo = 0;
int hi = kUtf8ComposeTableSize - 1;
while (lo <= hi) {
const int mid = (lo + hi) / 2;
const Utf8ComposeEntry& e = kUtf8ComposeTable[mid];
if (e.base < base || (e.base == base && e.mark < mark)) {
lo = mid + 1;
} else if (e.base > base || (e.base == base && e.mark > mark)) {
hi = mid - 1;
} else {
return e.composed;
}
}
return 0;
}
} // namespace
std::string utf8ComposeNfc(const std::string& in) {
// Fast path: NFC composition can only change text that contains a combining
// diacritical mark U+0300-036F (UTF-8 lead byte 0xCC or 0xCD). Plain ASCII and
// already-precomposed (NFC) text -- the vast majority of words -- have none, so
// return them untouched without walking codepoints or allocating. A 0xCD that is
// actually a non-combining codepoint just falls through to the full pass below.
bool maybeHasMarks = false;
for (const unsigned char c : in) {
if (c == 0xCC || c == 0xCD) {
maybeHasMarks = true;
break;
}
}
if (!maybeHasMarks) return in;
std::string out;
out.reserve(in.size());
const unsigned char* p = reinterpret_cast<const unsigned char*>(in.c_str());
uint32_t base = 0;
bool haveBase = false;
while (*p) {
const uint32_t cp = utf8NextCodepoint(&p);
if (cp == 0) break;
if (utf8IsCombiningMark(cp)) {
const uint32_t composed = haveBase ? utf8ComposePair(base, cp) : 0;
if (composed) {
base = composed; // keep accumulating further marks onto the composed char
continue;
}
// No composition: flush the pending base, then emit the mark unchanged.
if (haveBase) {
utf8AppendCodepoint(base, out);
haveBase = false;
}
utf8AppendCodepoint(cp, out);
} else {
if (haveBase) utf8AppendCodepoint(base, out);
base = cp;
haveBase = true;
}
}
if (haveBase) utf8AppendCodepoint(base, out);
return out;
}
int utf8CodepointLen(const unsigned char c) {
if (c < 0x80) return 1; // 0xxxxxxx
if ((c >> 5) == 0x6) return 2; // 110xxxxx
lib/Utf8/Utf8.h
+6
View File
@@ -12,6 +12,12 @@ size_t utf8RemoveLastChar(std::string& str);
// Truncate string by removing N UTF-8 codepoints from the end.
void utf8TruncateChars(std::string& str, size_t numChars);
// Canonical composition (NFC) for the Latin / Vietnamese range: precomposes a
// base letter followed by combining diacritical mark(s) into a single codepoint.
// Needed because the device fonts have no combining-mark positioning, so text
// stored in NFD (e.g. some EPUB chapter titles) otherwise renders broken.
std::string utf8ComposeNfc(const std::string& in);
// Truncate a raw char buffer to the last complete UTF-8 codepoint boundary.
// Returns the new length (<= len). If the buffer ends mid-sequence, the
// incomplete trailing bytes are excluded.
lib/Utf8/Utf8ComposeTable.h
+229
View File
@@ -0,0 +1,229 @@
// Auto-generated canonical composition (NFC) table for the Latin / Vietnamese
// range (combining marks U+0300-U+036F). Generated from Python unicodedata.
// Used by utf8ComposeNfc() to precompose decomposed (NFD) text so the device
// fonts (which lack combining-mark positioning) render it correctly.
#pragma once
#include <cstdint>
struct Utf8ComposeEntry {
uint16_t base;
uint16_t mark;
uint16_t composed;
};
// Sorted by (base, mark) for binary search.
static constexpr Utf8ComposeEntry kUtf8ComposeTable[] = {
{0x0041, 0x0300, 0x00C0}, {0x0041, 0x0301, 0x00C1}, {0x0041, 0x0302, 0x00C2}, {0x0041, 0x0303, 0x00C3},
{0x0041, 0x0304, 0x0100}, {0x0041, 0x0306, 0x0102}, {0x0041, 0x0307, 0x0226}, {0x0041, 0x0308, 0x00C4},
{0x0041, 0x0309, 0x1EA2}, {0x0041, 0x030A, 0x00C5}, {0x0041, 0x030C, 0x01CD}, {0x0041, 0x030F, 0x0200},
{0x0041, 0x0311, 0x0202}, {0x0041, 0x0323, 0x1EA0}, {0x0041, 0x0325, 0x1E00}, {0x0041, 0x0328, 0x0104},
{0x0041, 0x0340, 0x00C0}, {0x0041, 0x0341, 0x00C1}, {0x0042, 0x0307, 0x1E02}, {0x0042, 0x0323, 0x1E04},
{0x0042, 0x0331, 0x1E06}, {0x0043, 0x0301, 0x0106}, {0x0043, 0x0302, 0x0108}, {0x0043, 0x0307, 0x010A},
{0x0043, 0x030C, 0x010C}, {0x0043, 0x0327, 0x00C7}, {0x0043, 0x0341, 0x0106}, {0x0044, 0x0307, 0x1E0A},
{0x0044, 0x030C, 0x010E}, {0x0044, 0x0323, 0x1E0C}, {0x0044, 0x0327, 0x1E10}, {0x0044, 0x032D, 0x1E12},
{0x0044, 0x0331, 0x1E0E}, {0x0045, 0x0300, 0x00C8}, {0x0045, 0x0301, 0x00C9}, {0x0045, 0x0302, 0x00CA},
{0x0045, 0x0303, 0x1EBC}, {0x0045, 0x0304, 0x0112}, {0x0045, 0x0306, 0x0114}, {0x0045, 0x0307, 0x0116},
{0x0045, 0x0308, 0x00CB}, {0x0045, 0x0309, 0x1EBA}, {0x0045, 0x030C, 0x011A}, {0x0045, 0x030F, 0x0204},
{0x0045, 0x0311, 0x0206}, {0x0045, 0x0323, 0x1EB8}, {0x0045, 0x0327, 0x0228}, {0x0045, 0x0328, 0x0118},
{0x0045, 0x032D, 0x1E18}, {0x0045, 0x0330, 0x1E1A}, {0x0045, 0x0340, 0x00C8}, {0x0045, 0x0341, 0x00C9},
{0x0046, 0x0307, 0x1E1E}, {0x0047, 0x0301, 0x01F4}, {0x0047, 0x0302, 0x011C}, {0x0047, 0x0304, 0x1E20},
{0x0047, 0x0306, 0x011E}, {0x0047, 0x0307, 0x0120}, {0x0047, 0x030C, 0x01E6}, {0x0047, 0x0327, 0x0122},
{0x0047, 0x0341, 0x01F4}, {0x0048, 0x0302, 0x0124}, {0x0048, 0x0307, 0x1E22}, {0x0048, 0x0308, 0x1E26},
{0x0048, 0x030C, 0x021E}, {0x0048, 0x0323, 0x1E24}, {0x0048, 0x0327, 0x1E28}, {0x0048, 0x032E, 0x1E2A},
{0x0049, 0x0300, 0x00CC}, {0x0049, 0x0301, 0x00CD}, {0x0049, 0x0302, 0x00CE}, {0x0049, 0x0303, 0x0128},
{0x0049, 0x0304, 0x012A}, {0x0049, 0x0306, 0x012C}, {0x0049, 0x0307, 0x0130}, {0x0049, 0x0308, 0x00CF},
{0x0049, 0x0309, 0x1EC8}, {0x0049, 0x030C, 0x01CF}, {0x0049, 0x030F, 0x0208}, {0x0049, 0x0311, 0x020A},
{0x0049, 0x0323, 0x1ECA}, {0x0049, 0x0328, 0x012E}, {0x0049, 0x0330, 0x1E2C}, {0x0049, 0x0340, 0x00CC},
{0x0049, 0x0341, 0x00CD}, {0x0049, 0x0344, 0x1E2E}, {0x004A, 0x0302, 0x0134}, {0x004B, 0x0301, 0x1E30},
{0x004B, 0x030C, 0x01E8}, {0x004B, 0x0323, 0x1E32}, {0x004B, 0x0327, 0x0136}, {0x004B, 0x0331, 0x1E34},
{0x004B, 0x0341, 0x1E30}, {0x004C, 0x0301, 0x0139}, {0x004C, 0x030C, 0x013D}, {0x004C, 0x0323, 0x1E36},
{0x004C, 0x0327, 0x013B}, {0x004C, 0x032D, 0x1E3C}, {0x004C, 0x0331, 0x1E3A}, {0x004C, 0x0341, 0x0139},
{0x004D, 0x0301, 0x1E3E}, {0x004D, 0x0307, 0x1E40}, {0x004D, 0x0323, 0x1E42}, {0x004D, 0x0341, 0x1E3E},
{0x004E, 0x0300, 0x01F8}, {0x004E, 0x0301, 0x0143}, {0x004E, 0x0303, 0x00D1}, {0x004E, 0x0307, 0x1E44},
{0x004E, 0x030C, 0x0147}, {0x004E, 0x0323, 0x1E46}, {0x004E, 0x0327, 0x0145}, {0x004E, 0x032D, 0x1E4A},
{0x004E, 0x0331, 0x1E48}, {0x004E, 0x0340, 0x01F8}, {0x004E, 0x0341, 0x0143}, {0x004F, 0x0300, 0x00D2},
{0x004F, 0x0301, 0x00D3}, {0x004F, 0x0302, 0x00D4}, {0x004F, 0x0303, 0x00D5}, {0x004F, 0x0304, 0x014C},
{0x004F, 0x0306, 0x014E}, {0x004F, 0x0307, 0x022E}, {0x004F, 0x0308, 0x00D6}, {0x004F, 0x0309, 0x1ECE},
{0x004F, 0x030B, 0x0150}, {0x004F, 0x030C, 0x01D1}, {0x004F, 0x030F, 0x020C}, {0x004F, 0x0311, 0x020E},
{0x004F, 0x031B, 0x01A0}, {0x004F, 0x0323, 0x1ECC}, {0x004F, 0x0328, 0x01EA}, {0x004F, 0x0340, 0x00D2},
{0x004F, 0x0341, 0x00D3}, {0x0050, 0x0301, 0x1E54}, {0x0050, 0x0307, 0x1E56}, {0x0050, 0x0341, 0x1E54},
{0x0052, 0x0301, 0x0154}, {0x0052, 0x0307, 0x1E58}, {0x0052, 0x030C, 0x0158}, {0x0052, 0x030F, 0x0210},
{0x0052, 0x0311, 0x0212}, {0x0052, 0x0323, 0x1E5A}, {0x0052, 0x0327, 0x0156}, {0x0052, 0x0331, 0x1E5E},
{0x0052, 0x0341, 0x0154}, {0x0053, 0x0301, 0x015A}, {0x0053, 0x0302, 0x015C}, {0x0053, 0x0307, 0x1E60},
{0x0053, 0x030C, 0x0160}, {0x0053, 0x0323, 0x1E62}, {0x0053, 0x0326, 0x0218}, {0x0053, 0x0327, 0x015E},
{0x0053, 0x0341, 0x015A}, {0x0054, 0x0307, 0x1E6A}, {0x0054, 0x030C, 0x0164}, {0x0054, 0x0323, 0x1E6C},
{0x0054, 0x0326, 0x021A}, {0x0054, 0x0327, 0x0162}, {0x0054, 0x032D, 0x1E70}, {0x0054, 0x0331, 0x1E6E},
{0x0055, 0x0300, 0x00D9}, {0x0055, 0x0301, 0x00DA}, {0x0055, 0x0302, 0x00DB}, {0x0055, 0x0303, 0x0168},
{0x0055, 0x0304, 0x016A}, {0x0055, 0x0306, 0x016C}, {0x0055, 0x0308, 0x00DC}, {0x0055, 0x0309, 0x1EE6},
{0x0055, 0x030A, 0x016E}, {0x0055, 0x030B, 0x0170}, {0x0055, 0x030C, 0x01D3}, {0x0055, 0x030F, 0x0214},
{0x0055, 0x0311, 0x0216}, {0x0055, 0x031B, 0x01AF}, {0x0055, 0x0323, 0x1EE4}, {0x0055, 0x0324, 0x1E72},
{0x0055, 0x0328, 0x0172}, {0x0055, 0x032D, 0x1E76}, {0x0055, 0x0330, 0x1E74}, {0x0055, 0x0340, 0x00D9},
{0x0055, 0x0341, 0x00DA}, {0x0055, 0x0344, 0x01D7}, {0x0056, 0x0303, 0x1E7C}, {0x0056, 0x0323, 0x1E7E},
{0x0057, 0x0300, 0x1E80}, {0x0057, 0x0301, 0x1E82}, {0x0057, 0x0302, 0x0174}, {0x0057, 0x0307, 0x1E86},
{0x0057, 0x0308, 0x1E84}, {0x0057, 0x0323, 0x1E88}, {0x0057, 0x0340, 0x1E80}, {0x0057, 0x0341, 0x1E82},
{0x0058, 0x0307, 0x1E8A}, {0x0058, 0x0308, 0x1E8C}, {0x0059, 0x0300, 0x1EF2}, {0x0059, 0x0301, 0x00DD},
{0x0059, 0x0302, 0x0176}, {0x0059, 0x0303, 0x1EF8}, {0x0059, 0x0304, 0x0232}, {0x0059, 0x0307, 0x1E8E},
{0x0059, 0x0308, 0x0178}, {0x0059, 0x0309, 0x1EF6}, {0x0059, 0x0323, 0x1EF4}, {0x0059, 0x0340, 0x1EF2},
{0x0059, 0x0341, 0x00DD}, {0x005A, 0x0301, 0x0179}, {0x005A, 0x0302, 0x1E90}, {0x005A, 0x0307, 0x017B},
{0x005A, 0x030C, 0x017D}, {0x005A, 0x0323, 0x1E92}, {0x005A, 0x0331, 0x1E94}, {0x005A, 0x0341, 0x0179},
{0x0061, 0x0300, 0x00E0}, {0x0061, 0x0301, 0x00E1}, {0x0061, 0x0302, 0x00E2}, {0x0061, 0x0303, 0x00E3},
{0x0061, 0x0304, 0x0101}, {0x0061, 0x0306, 0x0103}, {0x0061, 0x0307, 0x0227}, {0x0061, 0x0308, 0x00E4},
{0x0061, 0x0309, 0x1EA3}, {0x0061, 0x030A, 0x00E5}, {0x0061, 0x030C, 0x01CE}, {0x0061, 0x030F, 0x0201},
{0x0061, 0x0311, 0x0203}, {0x0061, 0x0323, 0x1EA1}, {0x0061, 0x0325, 0x1E01}, {0x0061, 0x0328, 0x0105},
{0x0061, 0x0340, 0x00E0}, {0x0061, 0x0341, 0x00E1}, {0x0062, 0x0307, 0x1E03}, {0x0062, 0x0323, 0x1E05},
{0x0062, 0x0331, 0x1E07}, {0x0063, 0x0301, 0x0107}, {0x0063, 0x0302, 0x0109}, {0x0063, 0x0307, 0x010B},
{0x0063, 0x030C, 0x010D}, {0x0063, 0x0327, 0x00E7}, {0x0063, 0x0341, 0x0107}, {0x0064, 0x0307, 0x1E0B},
{0x0064, 0x030C, 0x010F}, {0x0064, 0x0323, 0x1E0D}, {0x0064, 0x0327, 0x1E11}, {0x0064, 0x032D, 0x1E13},
{0x0064, 0x0331, 0x1E0F}, {0x0065, 0x0300, 0x00E8}, {0x0065, 0x0301, 0x00E9}, {0x0065, 0x0302, 0x00EA},
{0x0065, 0x0303, 0x1EBD}, {0x0065, 0x0304, 0x0113}, {0x0065, 0x0306, 0x0115}, {0x0065, 0x0307, 0x0117},
{0x0065, 0x0308, 0x00EB}, {0x0065, 0x0309, 0x1EBB}, {0x0065, 0x030C, 0x011B}, {0x0065, 0x030F, 0x0205},
{0x0065, 0x0311, 0x0207}, {0x0065, 0x0323, 0x1EB9}, {0x0065, 0x0327, 0x0229}, {0x0065, 0x0328, 0x0119},
{0x0065, 0x032D, 0x1E19}, {0x0065, 0x0330, 0x1E1B}, {0x0065, 0x0340, 0x00E8}, {0x0065, 0x0341, 0x00E9},
{0x0066, 0x0307, 0x1E1F}, {0x0067, 0x0301, 0x01F5}, {0x0067, 0x0302, 0x011D}, {0x0067, 0x0304, 0x1E21},
{0x0067, 0x0306, 0x011F}, {0x0067, 0x0307, 0x0121}, {0x0067, 0x030C, 0x01E7}, {0x0067, 0x0327, 0x0123},
{0x0067, 0x0341, 0x01F5}, {0x0068, 0x0302, 0x0125}, {0x0068, 0x0307, 0x1E23}, {0x0068, 0x0308, 0x1E27},
{0x0068, 0x030C, 0x021F}, {0x0068, 0x0323, 0x1E25}, {0x0068, 0x0327, 0x1E29}, {0x0068, 0x032E, 0x1E2B},
{0x0068, 0x0331, 0x1E96}, {0x0069, 0x0300, 0x00EC}, {0x0069, 0x0301, 0x00ED}, {0x0069, 0x0302, 0x00EE},
{0x0069, 0x0303, 0x0129}, {0x0069, 0x0304, 0x012B}, {0x0069, 0x0306, 0x012D}, {0x0069, 0x0308, 0x00EF},
{0x0069, 0x0309, 0x1EC9}, {0x0069, 0x030C, 0x01D0}, {0x0069, 0x030F, 0x0209}, {0x0069, 0x0311, 0x020B},
{0x0069, 0x0323, 0x1ECB}, {0x0069, 0x0328, 0x012F}, {0x0069, 0x0330, 0x1E2D}, {0x0069, 0x0340, 0x00EC},
{0x0069, 0x0341, 0x00ED}, {0x0069, 0x0344, 0x1E2F}, {0x006A, 0x0302, 0x0135}, {0x006A, 0x030C, 0x01F0},
{0x006B, 0x0301, 0x1E31}, {0x006B, 0x030C, 0x01E9}, {0x006B, 0x0323, 0x1E33}, {0x006B, 0x0327, 0x0137},
{0x006B, 0x0331, 0x1E35}, {0x006B, 0x0341, 0x1E31}, {0x006C, 0x0301, 0x013A}, {0x006C, 0x030C, 0x013E},
{0x006C, 0x0323, 0x1E37}, {0x006C, 0x0327, 0x013C}, {0x006C, 0x032D, 0x1E3D}, {0x006C, 0x0331, 0x1E3B},
{0x006C, 0x0341, 0x013A}, {0x006D, 0x0301, 0x1E3F}, {0x006D, 0x0307, 0x1E41}, {0x006D, 0x0323, 0x1E43},
{0x006D, 0x0341, 0x1E3F}, {0x006E, 0x0300, 0x01F9}, {0x006E, 0x0301, 0x0144}, {0x006E, 0x0303, 0x00F1},
{0x006E, 0x0307, 0x1E45}, {0x006E, 0x030C, 0x0148}, {0x006E, 0x0323, 0x1E47}, {0x006E, 0x0327, 0x0146},
{0x006E, 0x032D, 0x1E4B}, {0x006E, 0x0331, 0x1E49}, {0x006E, 0x0340, 0x01F9}, {0x006E, 0x0341, 0x0144},
{0x006F, 0x0300, 0x00F2}, {0x006F, 0x0301, 0x00F3}, {0x006F, 0x0302, 0x00F4}, {0x006F, 0x0303, 0x00F5},
{0x006F, 0x0304, 0x014D}, {0x006F, 0x0306, 0x014F}, {0x006F, 0x0307, 0x022F}, {0x006F, 0x0308, 0x00F6},
{0x006F, 0x0309, 0x1ECF}, {0x006F, 0x030B, 0x0151}, {0x006F, 0x030C, 0x01D2}, {0x006F, 0x030F, 0x020D},
{0x006F, 0x0311, 0x020F}, {0x006F, 0x031B, 0x01A1}, {0x006F, 0x0323, 0x1ECD}, {0x006F, 0x0328, 0x01EB},
{0x006F, 0x0340, 0x00F2}, {0x006F, 0x0341, 0x00F3}, {0x0070, 0x0301, 0x1E55}, {0x0070, 0x0307, 0x1E57},
{0x0070, 0x0341, 0x1E55}, {0x0072, 0x0301, 0x0155}, {0x0072, 0x0307, 0x1E59}, {0x0072, 0x030C, 0x0159},
{0x0072, 0x030F, 0x0211}, {0x0072, 0x0311, 0x0213}, {0x0072, 0x0323, 0x1E5B}, {0x0072, 0x0327, 0x0157},
{0x0072, 0x0331, 0x1E5F}, {0x0072, 0x0341, 0x0155}, {0x0073, 0x0301, 0x015B}, {0x0073, 0x0302, 0x015D},
{0x0073, 0x0307, 0x1E61}, {0x0073, 0x030C, 0x0161}, {0x0073, 0x0323, 0x1E63}, {0x0073, 0x0326, 0x0219},
{0x0073, 0x0327, 0x015F}, {0x0073, 0x0341, 0x015B}, {0x0074, 0x0307, 0x1E6B}, {0x0074, 0x0308, 0x1E97},
{0x0074, 0x030C, 0x0165}, {0x0074, 0x0323, 0x1E6D}, {0x0074, 0x0326, 0x021B}, {0x0074, 0x0327, 0x0163},
{0x0074, 0x032D, 0x1E71}, {0x0074, 0x0331, 0x1E6F}, {0x0075, 0x0300, 0x00F9}, {0x0075, 0x0301, 0x00FA},
{0x0075, 0x0302, 0x00FB}, {0x0075, 0x0303, 0x0169}, {0x0075, 0x0304, 0x016B}, {0x0075, 0x0306, 0x016D},
{0x0075, 0x0308, 0x00FC}, {0x0075, 0x0309, 0x1EE7}, {0x0075, 0x030A, 0x016F}, {0x0075, 0x030B, 0x0171},
{0x0075, 0x030C, 0x01D4}, {0x0075, 0x030F, 0x0215}, {0x0075, 0x0311, 0x0217}, {0x0075, 0x031B, 0x01B0},
{0x0075, 0x0323, 0x1EE5}, {0x0075, 0x0324, 0x1E73}, {0x0075, 0x0328, 0x0173}, {0x0075, 0x032D, 0x1E77},
{0x0075, 0x0330, 0x1E75}, {0x0075, 0x0340, 0x00F9}, {0x0075, 0x0341, 0x00FA}, {0x0075, 0x0344, 0x01D8},
{0x0076, 0x0303, 0x1E7D}, {0x0076, 0x0323, 0x1E7F}, {0x0077, 0x0300, 0x1E81}, {0x0077, 0x0301, 0x1E83},
{0x0077, 0x0302, 0x0175}, {0x0077, 0x0307, 0x1E87}, {0x0077, 0x0308, 0x1E85}, {0x0077, 0x030A, 0x1E98},
{0x0077, 0x0323, 0x1E89}, {0x0077, 0x0340, 0x1E81}, {0x0077, 0x0341, 0x1E83}, {0x0078, 0x0307, 0x1E8B},
{0x0078, 0x0308, 0x1E8D}, {0x0079, 0x0300, 0x1EF3}, {0x0079, 0x0301, 0x00FD}, {0x0079, 0x0302, 0x0177},
{0x0079, 0x0303, 0x1EF9}, {0x0079, 0x0304, 0x0233}, {0x0079, 0x0307, 0x1E8F}, {0x0079, 0x0308, 0x00FF},
{0x0079, 0x0309, 0x1EF7}, {0x0079, 0x030A, 0x1E99}, {0x0079, 0x0323, 0x1EF5}, {0x0079, 0x0340, 0x1EF3},
{0x0079, 0x0341, 0x00FD}, {0x007A, 0x0301, 0x017A}, {0x007A, 0x0302, 0x1E91}, {0x007A, 0x0307, 0x017C},
{0x007A, 0x030C, 0x017E}, {0x007A, 0x0323, 0x1E93}, {0x007A, 0x0331, 0x1E95}, {0x007A, 0x0341, 0x017A},
{0x00A8, 0x0300, 0x1FED}, {0x00A8, 0x0301, 0x0385}, {0x00A8, 0x0340, 0x1FED}, {0x00A8, 0x0341, 0x0385},
{0x00A8, 0x0342, 0x1FC1}, {0x00C2, 0x0300, 0x1EA6}, {0x00C2, 0x0301, 0x1EA4}, {0x00C2, 0x0303, 0x1EAA},
{0x00C2, 0x0309, 0x1EA8}, {0x00C2, 0x0323, 0x1EAC}, {0x00C2, 0x0340, 0x1EA6}, {0x00C2, 0x0341, 0x1EA4},
{0x00C4, 0x0304, 0x01DE}, {0x00C5, 0x0301, 0x01FA}, {0x00C5, 0x0341, 0x01FA}, {0x00C6, 0x0301, 0x01FC},
{0x00C6, 0x0304, 0x01E2}, {0x00C6, 0x0341, 0x01FC}, {0x00C7, 0x0301, 0x1E08}, {0x00C7, 0x0341, 0x1E08},
{0x00CA, 0x0300, 0x1EC0}, {0x00CA, 0x0301, 0x1EBE}, {0x00CA, 0x0303, 0x1EC4}, {0x00CA, 0x0309, 0x1EC2},
{0x00CA, 0x0323, 0x1EC6}, {0x00CA, 0x0340, 0x1EC0}, {0x00CA, 0x0341, 0x1EBE}, {0x00CF, 0x0301, 0x1E2E},
{0x00CF, 0x0341, 0x1E2E}, {0x00D2, 0x031B, 0x1EDC}, {0x00D3, 0x031B, 0x1EDA}, {0x00D4, 0x0300, 0x1ED2},
{0x00D4, 0x0301, 0x1ED0}, {0x00D4, 0x0303, 0x1ED6}, {0x00D4, 0x0309, 0x1ED4}, {0x00D4, 0x0323, 0x1ED8},
{0x00D4, 0x0340, 0x1ED2}, {0x00D4, 0x0341, 0x1ED0}, {0x00D5, 0x0301, 0x1E4C}, {0x00D5, 0x0304, 0x022C},
{0x00D5, 0x0308, 0x1E4E}, {0x00D5, 0x031B, 0x1EE0}, {0x00D5, 0x0341, 0x1E4C}, {0x00D6, 0x0304, 0x022A},
{0x00D8, 0x0301, 0x01FE}, {0x00D8, 0x0341, 0x01FE}, {0x00D9, 0x031B, 0x1EEA}, {0x00DA, 0x031B, 0x1EE8},
{0x00DC, 0x0300, 0x01DB}, {0x00DC, 0x0301, 0x01D7}, {0x00DC, 0x0304, 0x01D5}, {0x00DC, 0x030C, 0x01D9},
{0x00DC, 0x0340, 0x01DB}, {0x00DC, 0x0341, 0x01D7}, {0x00E2, 0x0300, 0x1EA7}, {0x00E2, 0x0301, 0x1EA5},
{0x00E2, 0x0303, 0x1EAB}, {0x00E2, 0x0309, 0x1EA9}, {0x00E2, 0x0323, 0x1EAD}, {0x00E2, 0x0340, 0x1EA7},
{0x00E2, 0x0341, 0x1EA5}, {0x00E4, 0x0304, 0x01DF}, {0x00E5, 0x0301, 0x01FB}, {0x00E5, 0x0341, 0x01FB},
{0x00E6, 0x0301, 0x01FD}, {0x00E6, 0x0304, 0x01E3}, {0x00E6, 0x0341, 0x01FD}, {0x00E7, 0x0301, 0x1E09},
{0x00E7, 0x0341, 0x1E09}, {0x00EA, 0x0300, 0x1EC1}, {0x00EA, 0x0301, 0x1EBF}, {0x00EA, 0x0303, 0x1EC5},
{0x00EA, 0x0309, 0x1EC3}, {0x00EA, 0x0323, 0x1EC7}, {0x00EA, 0x0340, 0x1EC1}, {0x00EA, 0x0341, 0x1EBF},
{0x00EF, 0x0301, 0x1E2F}, {0x00EF, 0x0341, 0x1E2F}, {0x00F2, 0x031B, 0x1EDD}, {0x00F3, 0x031B, 0x1EDB},
{0x00F4, 0x0300, 0x1ED3}, {0x00F4, 0x0301, 0x1ED1}, {0x00F4, 0x0303, 0x1ED7}, {0x00F4, 0x0309, 0x1ED5},
{0x00F4, 0x0323, 0x1ED9}, {0x00F4, 0x0340, 0x1ED3}, {0x00F4, 0x0341, 0x1ED1}, {0x00F5, 0x0301, 0x1E4D},
{0x00F5, 0x0304, 0x022D}, {0x00F5, 0x0308, 0x1E4F}, {0x00F5, 0x031B, 0x1EE1}, {0x00F5, 0x0341, 0x1E4D},
{0x00F6, 0x0304, 0x022B}, {0x00F8, 0x0301, 0x01FF}, {0x00F8, 0x0341, 0x01FF}, {0x00F9, 0x031B, 0x1EEB},
{0x00FA, 0x031B, 0x1EE9}, {0x00FC, 0x0300, 0x01DC}, {0x00FC, 0x0301, 0x01D8}, {0x00FC, 0x0304, 0x01D6},
{0x00FC, 0x030C, 0x01DA}, {0x00FC, 0x0340, 0x01DC}, {0x00FC, 0x0341, 0x01D8}, {0x0102, 0x0300, 0x1EB0},
{0x0102, 0x0301, 0x1EAE}, {0x0102, 0x0303, 0x1EB4}, {0x0102, 0x0309, 0x1EB2}, {0x0102, 0x0323, 0x1EB6},
{0x0102, 0x0340, 0x1EB0}, {0x0102, 0x0341, 0x1EAE}, {0x0103, 0x0300, 0x1EB1}, {0x0103, 0x0301, 0x1EAF},
{0x0103, 0x0303, 0x1EB5}, {0x0103, 0x0309, 0x1EB3}, {0x0103, 0x0323, 0x1EB7}, {0x0103, 0x0340, 0x1EB1},
{0x0103, 0x0341, 0x1EAF}, {0x0106, 0x0327, 0x1E08}, {0x0107, 0x0327, 0x1E09}, {0x0112, 0x0300, 0x1E14},
{0x0112, 0x0301, 0x1E16}, {0x0112, 0x0340, 0x1E14}, {0x0112, 0x0341, 0x1E16}, {0x0113, 0x0300, 0x1E15},
{0x0113, 0x0301, 0x1E17}, {0x0113, 0x0340, 0x1E15}, {0x0113, 0x0341, 0x1E17}, {0x0114, 0x0327, 0x1E1C},
{0x0115, 0x0327, 0x1E1D}, {0x014C, 0x0300, 0x1E50}, {0x014C, 0x0301, 0x1E52}, {0x014C, 0x0328, 0x01EC},
{0x014C, 0x0340, 0x1E50}, {0x014C, 0x0341, 0x1E52}, {0x014D, 0x0300, 0x1E51}, {0x014D, 0x0301, 0x1E53},
{0x014D, 0x0328, 0x01ED}, {0x014D, 0x0340, 0x1E51}, {0x014D, 0x0341, 0x1E53}, {0x015A, 0x0307, 0x1E64},
{0x015B, 0x0307, 0x1E65}, {0x0160, 0x0307, 0x1E66}, {0x0161, 0x0307, 0x1E67}, {0x0168, 0x0301, 0x1E78},
{0x0168, 0x031B, 0x1EEE}, {0x0168, 0x0341, 0x1E78}, {0x0169, 0x0301, 0x1E79}, {0x0169, 0x031B, 0x1EEF},
{0x0169, 0x0341, 0x1E79}, {0x016A, 0x0308, 0x1E7A}, {0x016B, 0x0308, 0x1E7B}, {0x017F, 0x0307, 0x1E9B},
{0x01A0, 0x0300, 0x1EDC}, {0x01A0, 0x0301, 0x1EDA}, {0x01A0, 0x0303, 0x1EE0}, {0x01A0, 0x0309, 0x1EDE},
{0x01A0, 0x0323, 0x1EE2}, {0x01A0, 0x0340, 0x1EDC}, {0x01A0, 0x0341, 0x1EDA}, {0x01A1, 0x0300, 0x1EDD},
{0x01A1, 0x0301, 0x1EDB}, {0x01A1, 0x0303, 0x1EE1}, {0x01A1, 0x0309, 0x1EDF}, {0x01A1, 0x0323, 0x1EE3},
{0x01A1, 0x0340, 0x1EDD}, {0x01A1, 0x0341, 0x1EDB}, {0x01AF, 0x0300, 0x1EEA}, {0x01AF, 0x0301, 0x1EE8},
{0x01AF, 0x0303, 0x1EEE}, {0x01AF, 0x0309, 0x1EEC}, {0x01AF, 0x0323, 0x1EF0}, {0x01AF, 0x0340, 0x1EEA},
{0x01AF, 0x0341, 0x1EE8}, {0x01B0, 0x0300, 0x1EEB}, {0x01B0, 0x0301, 0x1EE9}, {0x01B0, 0x0303, 0x1EEF},
{0x01B0, 0x0309, 0x1EED}, {0x01B0, 0x0323, 0x1EF1}, {0x01B0, 0x0340, 0x1EEB}, {0x01B0, 0x0341, 0x1EE9},
{0x01B7, 0x030C, 0x01EE}, {0x01EA, 0x0304, 0x01EC}, {0x01EB, 0x0304, 0x01ED}, {0x0226, 0x0304, 0x01E0},
{0x0227, 0x0304, 0x01E1}, {0x0228, 0x0306, 0x1E1C}, {0x0229, 0x0306, 0x1E1D}, {0x022E, 0x0304, 0x0230},
{0x022F, 0x0304, 0x0231}, {0x0292, 0x030C, 0x01EF}, {0x0391, 0x0300, 0x1FBA}, {0x0391, 0x0301, 0x0386},
{0x0391, 0x0304, 0x1FB9}, {0x0391, 0x0306, 0x1FB8}, {0x0391, 0x0313, 0x1F08}, {0x0391, 0x0314, 0x1F09},
{0x0391, 0x0340, 0x1FBA}, {0x0391, 0x0341, 0x0386}, {0x0391, 0x0343, 0x1F08}, {0x0391, 0x0345, 0x1FBC},
{0x0395, 0x0300, 0x1FC8}, {0x0395, 0x0301, 0x0388}, {0x0395, 0x0313, 0x1F18}, {0x0395, 0x0314, 0x1F19},
{0x0395, 0x0340, 0x1FC8}, {0x0395, 0x0341, 0x0388}, {0x0395, 0x0343, 0x1F18}, {0x0397, 0x0300, 0x1FCA},
{0x0397, 0x0301, 0x0389}, {0x0397, 0x0313, 0x1F28}, {0x0397, 0x0314, 0x1F29}, {0x0397, 0x0340, 0x1FCA},
{0x0397, 0x0341, 0x0389}, {0x0397, 0x0343, 0x1F28}, {0x0397, 0x0345, 0x1FCC}, {0x0399, 0x0300, 0x1FDA},
{0x0399, 0x0301, 0x038A}, {0x0399, 0x0304, 0x1FD9}, {0x0399, 0x0306, 0x1FD8}, {0x0399, 0x0308, 0x03AA},
{0x0399, 0x0313, 0x1F38}, {0x0399, 0x0314, 0x1F39}, {0x0399, 0x0340, 0x1FDA}, {0x0399, 0x0341, 0x038A},
{0x0399, 0x0343, 0x1F38}, {0x039F, 0x0300, 0x1FF8}, {0x039F, 0x0301, 0x038C}, {0x039F, 0x0313, 0x1F48},
{0x039F, 0x0314, 0x1F49}, {0x039F, 0x0340, 0x1FF8}, {0x039F, 0x0341, 0x038C}, {0x039F, 0x0343, 0x1F48},
{0x03A1, 0x0314, 0x1FEC}, {0x03A5, 0x0300, 0x1FEA}, {0x03A5, 0x0301, 0x038E}, {0x03A5, 0x0304, 0x1FE9},
{0x03A5, 0x0306, 0x1FE8}, {0x03A5, 0x0308, 0x03AB}, {0x03A5, 0x0314, 0x1F59}, {0x03A5, 0x0340, 0x1FEA},
{0x03A5, 0x0341, 0x038E}, {0x03A9, 0x0300, 0x1FFA}, {0x03A9, 0x0301, 0x038F}, {0x03A9, 0x0313, 0x1F68},
{0x03A9, 0x0314, 0x1F69}, {0x03A9, 0x0340, 0x1FFA}, {0x03A9, 0x0341, 0x038F}, {0x03A9, 0x0343, 0x1F68},
{0x03A9, 0x0345, 0x1FFC}, {0x03AC, 0x0345, 0x1FB4}, {0x03AE, 0x0345, 0x1FC4}, {0x03B1, 0x0300, 0x1F70},
{0x03B1, 0x0301, 0x03AC}, {0x03B1, 0x0304, 0x1FB1}, {0x03B1, 0x0306, 0x1FB0}, {0x03B1, 0x0313, 0x1F00},
{0x03B1, 0x0314, 0x1F01}, {0x03B1, 0x0340, 0x1F70}, {0x03B1, 0x0341, 0x03AC}, {0x03B1, 0x0342, 0x1FB6},
{0x03B1, 0x0343, 0x1F00}, {0x03B1, 0x0345, 0x1FB3}, {0x03B5, 0x0300, 0x1F72}, {0x03B5, 0x0301, 0x03AD},
{0x03B5, 0x0313, 0x1F10}, {0x03B5, 0x0314, 0x1F11}, {0x03B5, 0x0340, 0x1F72}, {0x03B5, 0x0341, 0x03AD},
{0x03B5, 0x0343, 0x1F10}, {0x03B7, 0x0300, 0x1F74}, {0x03B7, 0x0301, 0x03AE}, {0x03B7, 0x0313, 0x1F20},
{0x03B7, 0x0314, 0x1F21}, {0x03B7, 0x0340, 0x1F74}, {0x03B7, 0x0341, 0x03AE}, {0x03B7, 0x0342, 0x1FC6},
{0x03B7, 0x0343, 0x1F20}, {0x03B7, 0x0345, 0x1FC3}, {0x03B9, 0x0300, 0x1F76}, {0x03B9, 0x0301, 0x03AF},
{0x03B9, 0x0304, 0x1FD1}, {0x03B9, 0x0306, 0x1FD0}, {0x03B9, 0x0308, 0x03CA}, {0x03B9, 0x0313, 0x1F30},
{0x03B9, 0x0314, 0x1F31}, {0x03B9, 0x0340, 0x1F76}, {0x03B9, 0x0341, 0x03AF}, {0x03B9, 0x0342, 0x1FD6},
{0x03B9, 0x0343, 0x1F30}, {0x03B9, 0x0344, 0x0390}, {0x03BF, 0x0300, 0x1F78}, {0x03BF, 0x0301, 0x03CC},
{0x03BF, 0x0313, 0x1F40}, {0x03BF, 0x0314, 0x1F41}, {0x03BF, 0x0340, 0x1F78}, {0x03BF, 0x0341, 0x03CC},
{0x03BF, 0x0343, 0x1F40}, {0x03C1, 0x0313, 0x1FE4}, {0x03C1, 0x0314, 0x1FE5}, {0x03C1, 0x0343, 0x1FE4},
{0x03C5, 0x0300, 0x1F7A}, {0x03C5, 0x0301, 0x03CD}, {0x03C5, 0x0304, 0x1FE1}, {0x03C5, 0x0306, 0x1FE0},
{0x03C5, 0x0308, 0x03CB}, {0x03C5, 0x0313, 0x1F50}, {0x03C5, 0x0314, 0x1F51}, {0x03C5, 0x0340, 0x1F7A},
{0x03C5, 0x0341, 0x03CD}, {0x03C5, 0x0342, 0x1FE6}, {0x03C5, 0x0343, 0x1F50}, {0x03C5, 0x0344, 0x03B0},
{0x03C9, 0x0300, 0x1F7C}, {0x03C9, 0x0301, 0x03CE}, {0x03C9, 0x0313, 0x1F60}, {0x03C9, 0x0314, 0x1F61},
{0x03C9, 0x0340, 0x1F7C}, {0x03C9, 0x0341, 0x03CE}, {0x03C9, 0x0342, 0x1FF6}, {0x03C9, 0x0343, 0x1F60},
{0x03C9, 0x0345, 0x1FF3}, {0x03CA, 0x0300, 0x1FD2}, {0x03CA, 0x0301, 0x0390}, {0x03CA, 0x0340, 0x1FD2},
{0x03CA, 0x0341, 0x0390}, {0x03CA, 0x0342, 0x1FD7}, {0x03CB, 0x0300, 0x1FE2}, {0x03CB, 0x0301, 0x03B0},
{0x03CB, 0x0340, 0x1FE2}, {0x03CB, 0x0341, 0x03B0}, {0x03CB, 0x0342, 0x1FE7}, {0x03CE, 0x0345, 0x1FF4},
{0x03D2, 0x0301, 0x03D3}, {0x03D2, 0x0308, 0x03D4}, {0x03D2, 0x0341, 0x03D3}, {0x0406, 0x0308, 0x0407},
{0x0410, 0x0306, 0x04D0}, {0x0410, 0x0308, 0x04D2}, {0x0413, 0x0301, 0x0403}, {0x0413, 0x0341, 0x0403},
{0x0415, 0x0300, 0x0400}, {0x0415, 0x0306, 0x04D6}, {0x0415, 0x0308, 0x0401}, {0x0415, 0x0340, 0x0400},
{0x0416, 0x0306, 0x04C1}, {0x0416, 0x0308, 0x04DC}, {0x0417, 0x0308, 0x04DE}, {0x0418, 0x0300, 0x040D},
{0x0418, 0x0304, 0x04E2}, {0x0418, 0x0306, 0x0419}, {0x0418, 0x0308, 0x04E4}, {0x0418, 0x0340, 0x040D},
{0x041A, 0x0301, 0x040C}, {0x041A, 0x0341, 0x040C}, {0x041E, 0x0308, 0x04E6}, {0x0423, 0x0304, 0x04EE},
{0x0423, 0x0306, 0x040E}, {0x0423, 0x0308, 0x04F0}, {0x0423, 0x030B, 0x04F2}, {0x0427, 0x0308, 0x04F4},
{0x042B, 0x0308, 0x04F8}, {0x042D, 0x0308, 0x04EC}, {0x0430, 0x0306, 0x04D1}, {0x0430, 0x0308, 0x04D3},
{0x0433, 0x0301, 0x0453}, {0x0433, 0x0341, 0x0453}, {0x0435, 0x0300, 0x0450}, {0x0435, 0x0306, 0x04D7},
{0x0435, 0x0308, 0x0451}, {0x0435, 0x0340, 0x0450}, {0x0436, 0x0306, 0x04C2}, {0x0436, 0x0308, 0x04DD},
{0x0437, 0x0308, 0x04DF}, {0x0438, 0x0300, 0x045D}, {0x0438, 0x0304, 0x04E3}, {0x0438, 0x0306, 0x0439},
{0x0438, 0x0308, 0x04E5}, {0x0438, 0x0340, 0x045D}, {0x043A, 0x0301, 0x045C}, {0x043A, 0x0341, 0x045C},
{0x043E, 0x0308, 0x04E7}, {0x0443, 0x0304, 0x04EF}, {0x0443, 0x0306, 0x045E}, {0x0443, 0x0308, 0x04F1},
{0x0443, 0x030B, 0x04F3}, {0x0447, 0x0308, 0x04F5}, {0x044B, 0x0308, 0x04F9}, {0x044D, 0x0308, 0x04ED},
{0x0456, 0x0308, 0x0457}, {0x0474, 0x030F, 0x0476}, {0x0475, 0x030F, 0x0477}, {0x04D8, 0x0308, 0x04DA},
{0x04D9, 0x0308, 0x04DB}, {0x04E8, 0x0308, 0x04EA}, {0x04E9, 0x0308, 0x04EB}, {0x1E36, 0x0304, 0x1E38},
{0x1E37, 0x0304, 0x1E39}, {0x1E5A, 0x0304, 0x1E5C}, {0x1E5B, 0x0304, 0x1E5D}, {0x1E60, 0x0323, 0x1E68},
{0x1E61, 0x0323, 0x1E69}, {0x1E62, 0x0307, 0x1E68}, {0x1E63, 0x0307, 0x1E69}, {0x1EA0, 0x0302, 0x1EAC},
{0x1EA0, 0x0306, 0x1EB6}, {0x1EA1, 0x0302, 0x1EAD}, {0x1EA1, 0x0306, 0x1EB7}, {0x1EB8, 0x0302, 0x1EC6},
{0x1EB9, 0x0302, 0x1EC7}, {0x1ECC, 0x0302, 0x1ED8}, {0x1ECC, 0x031B, 0x1EE2}, {0x1ECD, 0x0302, 0x1ED9},
{0x1ECD, 0x031B, 0x1EE3}, {0x1ECE, 0x031B, 0x1EDE}, {0x1ECF, 0x031B, 0x1EDF}, {0x1EE4, 0x031B, 0x1EF0},
{0x1EE5, 0x031B, 0x1EF1}, {0x1EE6, 0x031B, 0x1EEC}, {0x1EE7, 0x031B, 0x1EED},
};
static constexpr int kUtf8ComposeTableSize = sizeof(kUtf8ComposeTable) / sizeof(kUtf8ComposeTable[0]);
test/CMakeLists.txt
+1
View File
@@ -42,3 +42,4 @@ add_subdirectory(streaming_json_parser)
add_subdirectory(release_json_parser)
add_subdirectory(differential_rounding)
add_subdirectory(hyphenation_eval)
add_subdirectory(utf8_compose)
test/utf8_compose/CMakeLists.txt
+15
View File
@@ -0,0 +1,15 @@
add_executable(Utf8ComposeTest
Utf8ComposeTest.cpp
${REPO_ROOT}/lib/Utf8/Utf8.cpp
)
target_include_directories(Utf8ComposeTest PRIVATE
${REPO_ROOT}/lib/Utf8
)
target_link_libraries(Utf8ComposeTest PRIVATE
crosspoint_test_common
GTest::gtest_main
)
gtest_discover_tests(Utf8ComposeTest)
test/utf8_compose/Utf8ComposeTest.cpp
+54
View File
@@ -0,0 +1,54 @@
#include <gtest/gtest.h>
#include <string>
#include "Utf8.h"
namespace {
// Helpers to build NFD / expected byte sequences explicitly so the test does not
// depend on the encoding of this source file.
const std::string kCombGrave = "\xCC\x80"; // U+0300 COMBINING GRAVE ACCENT
const std::string kCombAcute = "\xCC\x81"; // U+0301 COMBINING ACUTE ACCENT
const std::string kCombCirc = "\xCC\x82"; // U+0302 COMBINING CIRCUMFLEX ACCENT
const std::string kCombDotBelow = "\xCC\xA3"; // U+0323 COMBINING DOT BELOW
} // namespace
// ASCII and already-precomposed (NFC) text must pass through untouched (fast path).
TEST(Utf8ComposeNfc, PassesThroughAsciiAndNfc) {
EXPECT_EQ(utf8ComposeNfc(""), "");
EXPECT_EQ(utf8ComposeNfc("hello world"), "hello world");
EXPECT_EQ(utf8ComposeNfc("caf\xC3\xA9"), "caf\xC3\xA9"); // é already U+00E9
}
// Single combining mark composes onto its base letter.
TEST(Utf8ComposeNfc, ComposesSingleMark) {
EXPECT_EQ(utf8ComposeNfc("e" + kCombAcute), "\xC3\xA9"); // e + ́ -> é (U+00E9)
EXPECT_EQ(utf8ComposeNfc("a" + kCombGrave), "\xC3\xA0"); // a + ̀ -> à (U+00E0)
}
// Vietnamese letters carry two stacked marks; composition must accumulate them
// onto the intermediate precomposed character (this is the crux of the feature).
TEST(Utf8ComposeNfc, ComposesStackedVietnameseMarks) {
// a + circumflex + acute -> ấ (U+1EA5)
EXPECT_EQ(utf8ComposeNfc("a" + kCombCirc + kCombAcute), "\xE1\xBA\xA5");
// a + dot-below + circumflex (canonical order) -> ậ (U+1EAD)
EXPECT_EQ(utf8ComposeNfc("a" + kCombDotBelow + kCombCirc), "\xE1\xBA\xAD");
}
// A combining mark with no composition for its base is left unchanged, and the
// base is preserved.
TEST(Utf8ComposeNfc, LeavesUncomposableMarksIntact) {
const std::string in = "q" + kCombAcute; // no precomposed "q with acute"
EXPECT_EQ(utf8ComposeNfc(in), in);
}
// A leading combining mark (no preceding base) is emitted unchanged.
TEST(Utf8ComposeNfc, HandlesLeadingMark) { EXPECT_EQ(utf8ComposeNfc(kCombAcute), kCombAcute); }
// Marks embedded in a longer word compose while surrounding text is preserved.
TEST(Utf8ComposeNfc, ComposesWithinWord) {
// "Ti" + e+circ+acute + "ng" -> "Tiếng"
EXPECT_EQ(utf8ComposeNfc("Ti" + std::string("e") + kCombCirc + kCombAcute + "ng"), "Ti\xE1\xBA\xBFng");
}