Embrace vietnamese preprocessing from crosspet

2026-04-30 15:07:04 +02:00
parent da7ad888c7
commit de62bcf591
3 changed files with 294 additions and 0 deletions
@@ -22,3 +22,14 @@ inline bool utf8IsCombiningMark(const uint32_t cp) {
         || (cp >= 0x20D0 && cp <= 0x20FF)   // Combining Diacritical Marks for Symbols
         || (cp >= 0xFE20 && cp <= 0xFE2F);  // Combining Half Marks
 }
+
+// Returns true for any combining mark relevant to Vietnamese NFC composition,
+// including U+031B (COMBINING HORN) which falls outside the standard mark range.
+inline bool utf8IsVietnameseCombining(const uint32_t cp) { return utf8IsCombiningMark(cp) || cp == 0x031B; }
+
+// Apply lightweight NFC-like normalization for Vietnamese precomposed characters.
+// Converts NFD sequences (base vowel + combining marks) into NFC precomposed
+// codepoints from the U+1EA0-U+1EF9 range. Safe no-op for already-NFC text.
+// Handles both canonical NFD ordering and the "natural" order produced by
+// macOS, Word, and older Vietnamese publishing tools.
+std::string utf8NfcNorm(std::string s);