From de62bcf591446d0cd14b31048570e5f6ce0fcada Mon Sep 17 00:00:00 2001 From: jpirnay Date: Thu, 30 Apr 2026 15:07:04 +0200 Subject: [PATCH] Embrace vietnamese preprocessing from crosspet --- lib/Epub/Epub/ParsedText.cpp | 1 + lib/Utf8/Utf8.cpp | 282 +++++++++++++++++++++++++++++++++++ lib/Utf8/Utf8.h | 11 ++ 3 files changed, 294 insertions(+) diff --git a/lib/Epub/Epub/ParsedText.cpp b/lib/Epub/Epub/ParsedText.cpp index c13a8e99..8ded07de 100644 --- a/lib/Epub/Epub/ParsedText.cpp +++ b/lib/Epub/Epub/ParsedText.cpp @@ -129,6 +129,7 @@ void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle, const bool attachToPrevious) { if (word.empty()) return; + word = utf8NfcNorm(std::move(word)); words.push_back(std::move(word)); EpdFontFamily::Style combinedStyle = fontStyle; if (underline) { diff --git a/lib/Utf8/Utf8.cpp b/lib/Utf8/Utf8.cpp index cdcc495b..db2ea5a2 100644 --- a/lib/Utf8/Utf8.cpp +++ b/lib/Utf8/Utf8.cpp @@ -92,3 +92,285 @@ void utf8TruncateChars(std::string& str, const size_t numChars) { utf8RemoveLastChar(str); } } + +// --------------------------------------------------------------------------- +// Vietnamese NFC normalization +// --------------------------------------------------------------------------- + +// Encode a Unicode codepoint as UTF-8 bytes appended to `out`. +static void appendCodepoint(std::string& out, const uint32_t cp) { + if (cp < 0x80u) { + out += static_cast(cp); + } else if (cp < 0x800u) { + out += static_cast(0xC0u | (cp >> 6)); + out += static_cast(0x80u | (cp & 0x3Fu)); + } else if (cp < 0x10000u) { + out += static_cast(0xE0u | (cp >> 12)); + out += static_cast(0x80u | ((cp >> 6) & 0x3Fu)); + out += static_cast(0x80u | (cp & 0x3Fu)); + } else { + out += static_cast(0xF0u | (cp >> 18)); + out += static_cast(0x80u | ((cp >> 12) & 0x3Fu)); + out += static_cast(0x80u | ((cp >> 6) & 0x3Fu)); + out += static_cast(0x80u | (cp & 0x3Fu)); + } +} + +// Lookup table: {(base << 16) | mark, composed_result}. +// Covers all Vietnamese precomposed characters U+1EA0-U+1EF9 plus the +// intermediate Latin composites (â ă ê ô ơ ư and uppercase matches). +// Handles both "natural" order (structural modifier before tone) and +// canonical NFD order (dot-below reordered before circumflex/breve). +// Table is sorted by key ascending for binary search. +struct ComposeEntry { + uint32_t key; // (base << 16) | mark + uint32_t result; +}; + +static const ComposeEntry kComposeTable[] = { + // Sorted ascending by key = (base << 16) | mark. + // Uppercase Latin (0x0041–0x0059) come before lowercase (0x0061–0x0079), + // then accented precomposed bases in codepoint order. + + // ── A (U+0041) ────────────────────────────────────────────────────── + {0x00410300u, 0x00C0u}, // A + grave → À + {0x00410301u, 0x00C1u}, // A + acute → Á + {0x00410302u, 0x00C2u}, // A + ^ → Â + {0x00410303u, 0x00C3u}, // A + tilde → Ã + {0x00410306u, 0x0102u}, // A + breve → Ă + {0x00410309u, 0x1EA2u}, // A + hook → Ả + {0x00410323u, 0x1EA0u}, // A + dot → Ạ + // ── E (U+0045) ────────────────────────────────────────────────────── + {0x00450300u, 0x00C8u}, // E + grave → È + {0x00450301u, 0x00C9u}, // E + acute → É + {0x00450302u, 0x00CAu}, // E + ^ → Ê + {0x00450303u, 0x1EBCu}, // E + tilde → Ẽ + {0x00450309u, 0x1EBAu}, // E + hook → Ẻ + {0x00450323u, 0x1EB8u}, // E + dot → Ẹ + // ── I (U+0049) ────────────────────────────────────────────────────── + {0x00490300u, 0x00CCu}, // I + grave → Ì + {0x00490301u, 0x00CDu}, // I + acute → Í + {0x00490303u, 0x0128u}, // I + tilde → Ĩ + {0x00490309u, 0x1EC8u}, // I + hook → Ỉ + {0x00490323u, 0x1ECAu}, // I + dot → Ị + // ── O (U+004F) ────────────────────────────────────────────────────── + {0x004F0300u, 0x00D2u}, // O + grave → Ò + {0x004F0301u, 0x00D3u}, // O + acute → Ó + {0x004F0302u, 0x00D4u}, // O + ^ → Ô + {0x004F0303u, 0x00D5u}, // O + tilde → Õ + {0x004F0309u, 0x1ECEu}, // O + hook → Ỏ + {0x004F031Bu, 0x01A0u}, // O + horn → Ơ + {0x004F0323u, 0x1ECCu}, // O + dot → Ọ + // ── U (U+0055) ────────────────────────────────────────────────────── + {0x00550300u, 0x00D9u}, // U + grave → Ù + {0x00550301u, 0x00DAu}, // U + acute → Ú + {0x00550303u, 0x0168u}, // U + tilde → Ũ + {0x00550309u, 0x1EE6u}, // U + hook → Ủ + {0x0055031Bu, 0x01AFu}, // U + horn → Ư + {0x00550323u, 0x1EE4u}, // U + dot → Ụ + // ── Y (U+0059) ────────────────────────────────────────────────────── + {0x00590300u, 0x1EF2u}, // Y + grave → Ỳ + {0x00590301u, 0x00DDu}, // Y + acute → Ý + {0x00590303u, 0x1EF8u}, // Y + tilde → Ỹ + {0x00590309u, 0x1EF6u}, // Y + hook → Ỷ + {0x00590323u, 0x1EF4u}, // Y + dot → Ỵ + // ── a (U+0061) ────────────────────────────────────────────────────── + {0x00610300u, 0x00E0u}, // a + grave → à + {0x00610301u, 0x00E1u}, // a + acute → á + {0x00610302u, 0x00E2u}, // a + ^ → â + {0x00610303u, 0x00E3u}, // a + tilde → ã + {0x00610306u, 0x0103u}, // a + breve → ă + {0x00610309u, 0x1EA3u}, // a + hook → ả + {0x00610323u, 0x1EA1u}, // a + dot → ạ + // ── e (U+0065) ────────────────────────────────────────────────────── + {0x00650300u, 0x00E8u}, // e + grave → è + {0x00650301u, 0x00E9u}, // e + acute → é + {0x00650302u, 0x00EAu}, // e + ^ → ê + {0x00650303u, 0x1EBDu}, // e + tilde → ẽ + {0x00650309u, 0x1EBBu}, // e + hook → ẻ + {0x00650323u, 0x1EB9u}, // e + dot → ẹ + // ── i (U+0069) ────────────────────────────────────────────────────── + {0x00690300u, 0x00ECu}, // i + grave → ì + {0x00690301u, 0x00EDu}, // i + acute → í + {0x00690303u, 0x0129u}, // i + tilde → ĩ + {0x00690309u, 0x1EC9u}, // i + hook → ỉ + {0x00690323u, 0x1ECBu}, // i + dot → ị + // ── o (U+006F) ────────────────────────────────────────────────────── + {0x006F0300u, 0x00F2u}, // o + grave → ò + {0x006F0301u, 0x00F3u}, // o + acute → ó + {0x006F0302u, 0x00F4u}, // o + ^ → ô + {0x006F0303u, 0x00F5u}, // o + tilde → õ + {0x006F0309u, 0x1ECFu}, // o + hook → ỏ + {0x006F031Bu, 0x01A1u}, // o + horn → ơ + {0x006F0323u, 0x1ECDu}, // o + dot → ọ + // ── u (U+0075) ────────────────────────────────────────────────────── + {0x00750300u, 0x00F9u}, // u + grave → ù + {0x00750301u, 0x00FAu}, // u + acute → ú + {0x00750303u, 0x0169u}, // u + tilde → ũ + {0x00750309u, 0x1EE7u}, // u + hook → ủ + {0x0075031Bu, 0x01B0u}, // u + horn → ư + {0x00750323u, 0x1EE5u}, // u + dot → ụ + // ── y (U+0079) ────────────────────────────────────────────────────── + {0x00790300u, 0x1EF3u}, // y + grave → ỳ + {0x00790301u, 0x00FDu}, // y + acute → ý + {0x00790303u, 0x1EF9u}, // y + tilde → ỹ + {0x00790309u, 0x1EF7u}, // y + hook → ỷ + {0x00790323u, 0x1EF5u}, // y + dot → ỵ + // ── Â (U+00C2) ────────────────────────────────────────────────────── + {0x00C20300u, 0x1EA6u}, // Â + grave → Ầ + {0x00C20301u, 0x1EA4u}, // Â + acute → Ấ + {0x00C20303u, 0x1EAAu}, // Â + tilde → Ẫ + {0x00C20309u, 0x1EA8u}, // Â + hook → Ẩ + {0x00C20323u, 0x1EACu}, // Â + dot → Ậ + // ── Ê (U+00CA) ────────────────────────────────────────────────────── + {0x00CA0300u, 0x1EC0u}, // Ê + grave → Ề + {0x00CA0301u, 0x1EBEu}, // Ê + acute → Ế + {0x00CA0303u, 0x1EC4u}, // Ê + tilde → Ễ + {0x00CA0309u, 0x1EC2u}, // Ê + hook → Ể + {0x00CA0323u, 0x1EC6u}, // Ê + dot → Ệ + // ── Ô (U+00D4) ────────────────────────────────────────────────────── + {0x00D40300u, 0x1ED2u}, // Ô + grave → Ồ + {0x00D40301u, 0x1ED0u}, // Ô + acute → Ố + {0x00D40303u, 0x1ED6u}, // Ô + tilde → Ỗ + {0x00D40309u, 0x1ED4u}, // Ô + hook → Ổ + {0x00D40323u, 0x1ED8u}, // Ô + dot → Ộ + // ── â (U+00E2) ────────────────────────────────────────────────────── + {0x00E20300u, 0x1EA7u}, // â + grave → ầ + {0x00E20301u, 0x1EA5u}, // â + acute → ấ + {0x00E20303u, 0x1EABu}, // â + tilde → ẫ + {0x00E20309u, 0x1EA9u}, // â + hook → ẩ + {0x00E20323u, 0x1EADu}, // â + dot → ậ + // ── ê (U+00EA) ────────────────────────────────────────────────────── + {0x00EA0300u, 0x1EC1u}, // ê + grave → ề + {0x00EA0301u, 0x1EBFu}, // ê + acute → ế + {0x00EA0303u, 0x1EC5u}, // ê + tilde → ễ + {0x00EA0309u, 0x1EC3u}, // ê + hook → ể + {0x00EA0323u, 0x1EC7u}, // ê + dot → ệ + // ── ô (U+00F4) ────────────────────────────────────────────────────── + {0x00F40300u, 0x1ED3u}, // ô + grave → ồ + {0x00F40301u, 0x1ED1u}, // ô + acute → ố + {0x00F40303u, 0x1ED7u}, // ô + tilde → ỗ + {0x00F40309u, 0x1ED5u}, // ô + hook → ổ + {0x00F40323u, 0x1ED9u}, // ô + dot → ộ + // ── Ă (U+0102) ────────────────────────────────────────────────────── + {0x01020300u, 0x1EB0u}, // Ă + grave → Ằ + {0x01020301u, 0x1EAEu}, // Ă + acute → Ắ + {0x01020303u, 0x1EB4u}, // Ă + tilde → Ẵ + {0x01020309u, 0x1EB2u}, // Ă + hook → Ẳ + {0x01020323u, 0x1EB6u}, // Ă + dot → Ặ + // ── ă (U+0103) ────────────────────────────────────────────────────── + {0x01030300u, 0x1EB1u}, // ă + grave → ằ + {0x01030301u, 0x1EAFu}, // ă + acute → ắ + {0x01030303u, 0x1EB5u}, // ă + tilde → ẵ + {0x01030309u, 0x1EB3u}, // ă + hook → ẳ + {0x01030323u, 0x1EB7u}, // ă + dot → ặ + // ── Ơ (U+01A0) ────────────────────────────────────────────────────── + {0x01A00300u, 0x1EDCu}, // Ơ + grave → Ờ + {0x01A00301u, 0x1EDAu}, // Ơ + acute → Ớ + {0x01A00303u, 0x1EE0u}, // Ơ + tilde → Ỡ + {0x01A00309u, 0x1EDEu}, // Ơ + hook → Ở + {0x01A00323u, 0x1EE2u}, // Ơ + dot → Ợ + // ── ơ (U+01A1) ────────────────────────────────────────────────────── + {0x01A10300u, 0x1EDDu}, // ơ + grave → ờ + {0x01A10301u, 0x1EDBu}, // ơ + acute → ớ + {0x01A10303u, 0x1EE1u}, // ơ + tilde → ỡ + {0x01A10309u, 0x1EDFu}, // ơ + hook → ở + {0x01A10323u, 0x1EE3u}, // ơ + dot → ợ + // ── Ư (U+01AF) ────────────────────────────────────────────────────── + {0x01AF0300u, 0x1EEAu}, // Ư + grave → Ừ + {0x01AF0301u, 0x1EE8u}, // Ư + acute → Ứ + {0x01AF0303u, 0x1EEEu}, // Ư + tilde → Ữ + {0x01AF0309u, 0x1EECu}, // Ư + hook → Ử + {0x01AF0323u, 0x1EF0u}, // Ư + dot → Ự + // ── ư (U+01B0) ────────────────────────────────────────────────────── + {0x01B00300u, 0x1EEBu}, // ư + grave → ừ + {0x01B00301u, 0x1EE9u}, // ư + acute → ứ + {0x01B00303u, 0x1EEFu}, // ư + tilde → ữ + {0x01B00309u, 0x1EEDu}, // ư + hook → ử + {0x01B00323u, 0x1EF1u}, // ư + dot → ự + // ── NFD canonical-order recomposition ──────────────────────────────── + // In NFD, dot-below (CCC 220) sorts BEFORE circumflex/breve (CCC 230). + // So ậ = a+U+0323+U+0302, ặ = a+U+0323+U+0306, ệ = e+U+0323+U+0302, etc. + // Greedy first step (a+0323=ạ, e+0323=ẹ, o+0323=ọ) then needs: + {0x1EA00302u, 0x1EACu}, // Ạ + ^ → Ậ + {0x1EA00306u, 0x1EB6u}, // Ạ + breve → Ặ + {0x1EA10302u, 0x1EADu}, // ạ + ^ → ậ + {0x1EA10306u, 0x1EB7u}, // ạ + breve → ặ + {0x1EB80302u, 0x1EC6u}, // Ẹ + ^ → Ệ + {0x1EB90302u, 0x1EC7u}, // ẹ + ^ → ệ + {0x1ECC0302u, 0x1ED8u}, // Ọ + ^ → Ộ + {0x1ECD0302u, 0x1ED9u}, // ọ + ^ → ộ +}; + +static constexpr size_t kComposeTableSize = sizeof(kComposeTable) / sizeof(kComposeTable[0]); + +// Binary search for (base, mark) composition. Returns 0 if no entry found. +static uint32_t composeOne(const uint32_t base, const uint32_t mark) { + if (base > 0xFFFFu || mark > 0xFFFFu) return 0u; + const uint32_t key = (base << 16u) | mark; + size_t lo = 0; + size_t hi = kComposeTableSize; + while (lo < hi) { + const size_t mid = lo + (hi - lo) / 2; + if (kComposeTable[mid].key == key) return kComposeTable[mid].result; + if (kComposeTable[mid].key < key) + lo = mid + 1; + else + hi = mid; + } + return 0u; +} + +std::string utf8NfcNorm(std::string s) { + if (s.empty()) return s; + + // Fast-path: skip strings with no combining marks (most already-NFC text). + bool hasCombining = false; + { + const auto* p = reinterpret_cast(s.c_str()); + uint32_t cp; + while ((cp = utf8NextCodepoint(&p))) { + if (utf8IsVietnameseCombining(cp)) { + hasCombining = true; + break; + } + } + } + if (!hasCombining) return s; + + std::string out; + out.reserve(s.size()); + + const auto* ptr = reinterpret_cast(s.c_str()); + while (*ptr != '\0') { + uint32_t base = utf8NextCodepoint(&ptr); + if (base == 0) break; + + // Greedily absorb following Vietnamese combining marks into base. + while (*ptr != '\0') { + const auto* markStart = ptr; + const uint32_t mark = utf8NextCodepoint(&ptr); + if (mark == 0) break; + + if (!utf8IsVietnameseCombining(mark)) { + // Non-combining: put it back and stop absorbing. + ptr = markStart; + break; + } + + const uint32_t composed = composeOne(base, mark); + if (composed != 0u) { + base = composed; // absorbed — try next mark + } else { + // Cannot compose: emit current base as-is, the unmatched mark becomes + // the new base token (it will be emitted or further composed next loop). + appendCodepoint(out, base); + base = mark; + } + } + + appendCodepoint(out, base); + } + + return out; +} diff --git a/lib/Utf8/Utf8.h b/lib/Utf8/Utf8.h index a3310107..3a33f4db 100644 --- a/lib/Utf8/Utf8.h +++ b/lib/Utf8/Utf8.h @@ -22,3 +22,14 @@ inline bool utf8IsCombiningMark(const uint32_t cp) { || (cp >= 0x20D0 && cp <= 0x20FF) // Combining Diacritical Marks for Symbols || (cp >= 0xFE20 && cp <= 0xFE2F); // Combining Half Marks } + +// Returns true for any combining mark relevant to Vietnamese NFC composition, +// including U+031B (COMBINING HORN) which falls outside the standard mark range. +inline bool utf8IsVietnameseCombining(const uint32_t cp) { return utf8IsCombiningMark(cp) || cp == 0x031B; } + +// Apply lightweight NFC-like normalization for Vietnamese precomposed characters. +// Converts NFD sequences (base vowel + combining marks) into NFC precomposed +// codepoints from the U+1EA0-U+1EF9 range. Safe no-op for already-NFC text. +// Handles both canonical NFD ordering and the "natural" order produced by +// macOS, Word, and older Vietnamese publishing tools. +std::string utf8NfcNorm(std::string s);