feat: Arabic/Farsi/Urdu bidi reordering and contextual shaping — PR 1/3 (#2541)
Co-authored-by: Uri Tauber <uritaube@gmail.com>
This commit is contained in:
Husam Younis
co-authored by
Uri Tauber
Uri Tauber
parent
552b2683e6
commit
932a472835
@@ -11,6 +11,13 @@ extern "C" {
|
||||
#include <Utf8.h>
|
||||
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
|
||||
// Guards the static bidi_char buffers in applyBidiVisual() and
|
||||
// computeVisualWordOrder(). The bidi+shaping pipeline is not reentrant;
|
||||
// this mutex serialises access so multi-core callers don't corrupt each
|
||||
// other's intermediate state.
|
||||
static std::mutex bidiMutex;
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -68,11 +75,25 @@ int detectParagraphLevel(const char* utf8, const int fallbackLevel, const int ma
|
||||
return fallbackLevel & 1;
|
||||
}
|
||||
|
||||
bool isTransparentMark(const uint32_t cp) {
|
||||
// RTL-script combining marks: Hebrew niqqud/cantillation and Arabic
|
||||
// harakat/Quranic annotation. Transparent for Arabic joining (do_shape
|
||||
// skips them), zero-advance for measurement, and rendered as overlays on
|
||||
// the preceding base glyph when the active font carries their glyphs.
|
||||
// The cp >= 0x0591 guard keeps Latin combining marks (U+0300-U+036F, also
|
||||
// NSM) on their existing utf8IsCombiningMark() rendering path.
|
||||
return cp >= 0x0591 && bidi_class(cp) == NSM;
|
||||
}
|
||||
|
||||
bool applyBidiVisual(const char* utf8, std::string& out, int paragraphLevel) {
|
||||
if (!utf8 || !*utf8) return false;
|
||||
const std::lock_guard<std::mutex> lock(bidiMutex);
|
||||
|
||||
static bidi_char line[BIDI_MAX_LINE];
|
||||
static bidi_char shaped[BIDI_MAX_LINE];
|
||||
int count = 0;
|
||||
int lastBase = -1; // last non-formatter character (mintty's ibase)
|
||||
uint8_t pendingJoiners = 0; // ZWJ/ZWNJ seen since lastBase
|
||||
auto* p = reinterpret_cast<const unsigned char*>(utf8);
|
||||
while (*p) {
|
||||
if (count >= BIDI_MAX_LINE) {
|
||||
@@ -84,18 +105,66 @@ bool applyBidiVisual(const char* utf8, std::string& out, int paragraphLevel) {
|
||||
if (!cp || cp == REPLACEMENT_GLYPH) break;
|
||||
line[count].origwc = line[count].wc = cp;
|
||||
line[count].index = static_cast<uint16_t>(count);
|
||||
line[count].joiners = 0;
|
||||
|
||||
// Flag Arabic joining formatters mintty-style (termline.c): the ZWJ/ZWNJ
|
||||
// goes into the low nibble of the character it follows and the high
|
||||
// nibble of the character it precedes. Flags are assigned in logical
|
||||
// order here; do_shape() reads them after reordering.
|
||||
if (cp == 0x200C || cp == 0x200D) {
|
||||
const uint8_t joiner = (cp == 0x200D) ? ZWJ : ZWNJ;
|
||||
if (lastBase >= 0) line[lastBase].joiners |= joiner;
|
||||
pendingJoiners |= joiner;
|
||||
} else {
|
||||
line[count].joiners = pendingJoiners << 4;
|
||||
pendingJoiners = 0;
|
||||
lastBase = count;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
if (!count) return false;
|
||||
|
||||
const bool autodir = (paragraphLevel < 0);
|
||||
const int level = autodir ? 0 : (paragraphLevel & 1);
|
||||
|
||||
// Order matters (mintty does the same): do_bidi() first to obtain visual
|
||||
// order, then do_shape() — contextual forms are resolved from *visual*
|
||||
// adjacency, and shaping presentation forms must never be reordered.
|
||||
do_bidi(autodir, level, line, count);
|
||||
do_shape(line, shaped, count);
|
||||
|
||||
out.clear();
|
||||
out.reserve(std::strlen(utf8));
|
||||
// Lam-Alef collapse sentinel and zero-width joining formatters have done
|
||||
// their job during shaping and have no glyphs to render.
|
||||
const auto filtered = [](const uint32_t cp) { return cp == LIGATURE_PLACEHOLDER || cp == 0x200C || cp == 0x200D; };
|
||||
for (int i = 0; i < count; i++) {
|
||||
utf8AppendCodepoint(line[i].wc, out);
|
||||
const uint32_t cp = shaped[i].wc;
|
||||
if (filtered(cp)) continue;
|
||||
if (!isTransparentMark(cp)) {
|
||||
utf8AppendCodepoint(cp, out);
|
||||
continue;
|
||||
}
|
||||
// UAX#9 rule L3: reversing an RTL run leaves combining marks *before*
|
||||
// their base character. The renderer overlays a mark on the most
|
||||
// recently drawn glyph, so emit the base first, then its marks in
|
||||
// logical order. `index` is the original logical position: a base
|
||||
// following its marks with a *lower* index means the run was reversed.
|
||||
int j = i; // [i, j) = the run of marks (and filtered entries)
|
||||
while (j < count && (filtered(shaped[j].wc) || isTransparentMark(shaped[j].wc))) j++;
|
||||
if (j < count && shaped[j].index < shaped[i].index) {
|
||||
utf8AppendCodepoint(shaped[j].wc, out);
|
||||
for (int k = j - 1; k >= i; k--) {
|
||||
if (isTransparentMark(shaped[k].wc)) utf8AppendCodepoint(shaped[k].wc, out);
|
||||
}
|
||||
i = j; // base already emitted
|
||||
} else {
|
||||
// Unreversed (or trailing, base-less) marks already follow their base.
|
||||
for (int k = i; k < j; k++) {
|
||||
if (isTransparentMark(shaped[k].wc)) utf8AppendCodepoint(shaped[k].wc, out);
|
||||
}
|
||||
i = j - 1;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -105,6 +174,7 @@ bool computeVisualWordOrder(const std::vector<std::string>& words, bool paragrap
|
||||
visualOrder.clear();
|
||||
const size_t nWords = words.size();
|
||||
if (nWords <= 1 || nWords > BIDI_MAX_LINE) return false;
|
||||
const std::lock_guard<std::mutex> lock(bidiMutex);
|
||||
|
||||
static bidi_char line[BIDI_MAX_LINE];
|
||||
int count = 0;
|
||||
@@ -121,6 +191,7 @@ bool computeVisualWordOrder(const std::vector<std::string>& words, bool paragrap
|
||||
if (!cp || cp == REPLACEMENT_GLYPH) break;
|
||||
line[count].origwc = line[count].wc = cp;
|
||||
line[count].index = static_cast<uint16_t>(w);
|
||||
line[count].joiners = 0;
|
||||
count++;
|
||||
}
|
||||
|
||||
@@ -131,6 +202,7 @@ bool computeVisualWordOrder(const std::vector<std::string>& words, bool paragrap
|
||||
}
|
||||
line[count].origwc = line[count].wc = ' ';
|
||||
line[count].index = static_cast<uint16_t>(nWords);
|
||||
line[count].joiners = 0;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,14 @@ bool startsWithRtl(const char* utf8, int maxStrongChars = RTL_PARAGRAPH_PROBE_DE
|
||||
|
||||
int detectParagraphLevel(const char* utf8, int fallbackLevel = 0, int maxStrongChars = 64);
|
||||
|
||||
// True for RTL-script non-spacing marks (Hebrew niqqud/cantillation, Arabic
|
||||
// harakat and Quranic annotation): zero-width for measurement, transparent for
|
||||
// Arabic joining, and rendered as overlays on the preceding base glyph when
|
||||
// the active font carries their glyphs (SD fonts; built-in fonts don't).
|
||||
// Latin combining marks (U+0300-U+036F) intentionally return false — they are
|
||||
// handled by the utf8IsCombiningMark() rendering path.
|
||||
bool isTransparentMark(uint32_t cp);
|
||||
|
||||
// paragraphLevel: -1 = auto-detect, 0 = LTR, 1 = RTL
|
||||
bool applyBidiVisual(const char* utf8, std::string& out, int paragraphLevel = -1);
|
||||
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
/* bidiclasses.t — bidi class table for CrossPoint Hebrew/English epub.
|
||||
/* bidiclasses.t — bidi class table for CrossPoint RTL (Hebrew/Arabic) epub.
|
||||
*
|
||||
* Coverage rationale:
|
||||
* Hebrew + English is the primary target. However, CrossPoint renders
|
||||
* Latin and Cyrillic scripts for many other languages, so these MUST be
|
||||
* classified as L (not fall through to ON) to avoid regression when they
|
||||
* appear adjacent to Hebrew runs.
|
||||
* Hebrew and Arabic-script languages (Arabic, Farsi, Urdu, Sindhi, Pashto,
|
||||
* Kurdish) are the RTL targets. CrossPoint also renders Latin and Cyrillic
|
||||
* scripts for many other languages, so these MUST be classified as L (not
|
||||
* fall through to ON) to avoid regression when they appear adjacent to
|
||||
* RTL runs.
|
||||
*
|
||||
* Scripts NOT in this table fall through to ON — correct per UAX#9 for
|
||||
* scripts CrossPoint's fonts don't support (CJK, Arabic, Devanagari, etc.)
|
||||
* scripts CrossPoint's fonts don't support (CJK, Devanagari, etc.)
|
||||
* ON is the right class for "unknown" — it behaves neutrally.
|
||||
*
|
||||
* Arabic ranges are sourced from Unicode UCD extracted/DerivedBidiClass.txt
|
||||
* (values verified against Unicode 17.0.0).
|
||||
*
|
||||
* Entries sorted ascending by first (binary search requirement).
|
||||
*/
|
||||
|
||||
@@ -84,6 +88,36 @@
|
||||
{0x05D0, 0x05EA, R}, /* alef … tav */
|
||||
{0x05F0, 0x05F4, R}, /* alternative forms + geresh/gershayim */
|
||||
|
||||
/* ── Arabic / Perso-Arabic (DerivedBidiClass.txt) ───────────────────── */
|
||||
/* Letters are AL (Arabic Letter), harakat/marks are NSM, Arabic-Indic
|
||||
digits are AN, extended (Farsi/Urdu) digits are EN.
|
||||
0x0606-0x0607, 0x060E-0x060F, 0x06DE, 0x06E9 are ON — fall through. */
|
||||
{0x0600, 0x0605, AN}, /* Arabic number signs (Cf) */
|
||||
{0x0608, 0x0608, AL}, /* Arabic ray */
|
||||
{0x0609, 0x060A, ET}, /* per mille / per ten thousand */
|
||||
{0x060B, 0x060B, AL}, /* afghani sign */
|
||||
{0x060C, 0x060C, CS}, /* Arabic comma */
|
||||
{0x060D, 0x060D, AL}, /* Arabic date separator */
|
||||
{0x0610, 0x061A, NSM}, /* honorifics + small marks */
|
||||
{0x061B, 0x061F, AL}, /* semicolon, ALM, end-of-text, hamza mark, question mark */
|
||||
{0x0620, 0x064A, AL}, /* core Arabic letters + tatweel */
|
||||
{0x064B, 0x065F, NSM}, /* harakat (fathatan … wavy hamza below) */
|
||||
{0x0660, 0x0669, AN}, /* Arabic-Indic digits ٠-٩ */
|
||||
{0x066A, 0x066A, ET}, /* Arabic percent sign */
|
||||
{0x066B, 0x066C, AN}, /* decimal / thousands separators */
|
||||
{0x066D, 0x066F, AL}, /* five-pointed star, dotless beh/qaf */
|
||||
{0x0670, 0x0670, NSM}, /* superscript alef */
|
||||
{0x0671, 0x06D5, AL}, /* extended letters: Farsi, Urdu, Sindhi, Pashto, Kurdish */
|
||||
{0x06D6, 0x06DC, NSM}, /* Quranic annotation marks */
|
||||
{0x06DD, 0x06DD, AN}, /* end of ayah */
|
||||
{0x06DF, 0x06E4, NSM},
|
||||
{0x06E5, 0x06E6, AL}, /* small waw / small yeh */
|
||||
{0x06E7, 0x06E8, NSM},
|
||||
{0x06EA, 0x06ED, NSM},
|
||||
{0x06EE, 0x06EF, AL}, /* dal/reh with inverted V */
|
||||
{0x06F0, 0x06F9, EN}, /* extended Arabic-Indic digits ۰-۹ (Farsi/Urdu) — EN per UCD */
|
||||
{0x06FA, 0x06FF, AL},
|
||||
|
||||
/* ── Latin Extended Additional (L) ─────────────────────────────────── */
|
||||
/* Covers accented chars for Vietnamese, Welsh, Romanian, etc.
|
||||
Not currently rendered by CrossPoint fonts, but costs only 2 table rows. */
|
||||
@@ -110,6 +144,14 @@
|
||||
{0x2069, 0x2069, PDI},
|
||||
{0x206A, 0x206F, BN},
|
||||
|
||||
/* ── Arabic presentation forms (output of do_shape()) ───────────────── */
|
||||
/* Contextual/ligature forms emitted by the shaper must classify as AL so
|
||||
a reshaped line still resolves RTL. Ranges per DerivedBidiClass.txt;
|
||||
0xFBC3-0xFBD2 are ON — fall through. */
|
||||
{0xFB50, 0xFBC2, AL}, /* Presentation Forms-A: Perso-Arabic contextual forms */
|
||||
{0xFBD3, 0xFBFF, AL}, /* Presentation Forms-A: NG … Farsi Yeh forms */
|
||||
{0xFE70, 0xFE74, AL}, /* Presentation Forms-B: harakat isolated forms */
|
||||
{0xFE76, 0xFEFC, AL}, /* Presentation Forms-B: contextual forms + Lam-Alef ligatures */
|
||||
|
||||
/* ── Byte Order Mark ────────────────────────────────────────────────── */
|
||||
{0xFEFF, 0xFEFF, BN},
|
||||
|
||||
@@ -127,6 +127,345 @@ ucschar mirror(ucschar c) {
|
||||
return p ? p->to : c;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════
|
||||
* Arabic contextual shaping — do_shape()
|
||||
*
|
||||
* Ported from mintty src/minibidi.c (https://github.com/mintty/mintty),
|
||||
* original author Ahmad Khalifa (www.arabeyes.org), maintained by
|
||||
* Thomas Wolff. MIT licence.
|
||||
*
|
||||
* CrossPoint deviations from the upstream code, each marked inline:
|
||||
* 1. Joining-context lookups skip NSM marks (harakat). mintty stores
|
||||
* combining marks out-of-line in terminal cells, so they never appear
|
||||
* in its bidi_char array; in CrossPoint they are real array entries
|
||||
* and must be transparent for joining (Unicode joining type T).
|
||||
* 2. The Alef absorbed into a Lam-Alef ligature is overwritten with
|
||||
* LIGATURE_PLACEHOLDER instead of a space: a terminal must keep the
|
||||
* cell, a proportional-text renderer must drop the character.
|
||||
* 3. The STYPE/SISOLATED macros became functions backed by a second
|
||||
* lookup table covering Perso-Arabic letters outside mintty's native
|
||||
* U+0621–U+064A range (Farsi پ چ ژ گ, Urdu ٹ ڈ ڑ ں ہ ے, plus Sindhi/
|
||||
* Pashto/Kurdish letters). Joining types are sourced from Unicode
|
||||
* ArabicShaping.txt and presentation forms from UnicodeData.txt
|
||||
* (Arabic Presentation Forms-A), both Unicode 17.0.0. Letters with a
|
||||
* joining type but no presentation-form codepoints keep their base
|
||||
* codepoint (neighbours still shape correctly around them).
|
||||
* U+200C/U+200D also get their ArabicShaping.txt types (U and C) so
|
||||
* that in-stream ZWJ/ZWNJ — which mintty never has in its array —
|
||||
* affect adjacency the same way the joiners flags do.
|
||||
* ═══════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* Shaping Types (mintty) */
|
||||
enum {
|
||||
SL, /* Left-Joining, doesn't exist in U+0600 - U+06FF */
|
||||
SR, /* Right-Joining, i.e. has Isolated, Final */
|
||||
SD, /* Dual-Joining, i.e. has Isolated, Final, Initial, Medial */
|
||||
SU, /* Non-Joining */
|
||||
SC /* Join-Causing, like U+0640 (TATWEEL) */
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
uchar type;
|
||||
uchar form_b; /* isolated form = 0xFE00 + form_b (Presentation Forms-B) */
|
||||
} shape_node;
|
||||
|
||||
/* Kept near the actual table, for verification. (mintty) */
|
||||
enum { SHAPE_FIRST = 0x621, SHAPE_LAST = 0x64A };
|
||||
|
||||
/* mintty's shapetypes[] — verbatim */
|
||||
static const shape_node shapetypes[] = {
|
||||
/* index, Typ, Iso, Ligature Index */
|
||||
/* 621 */ {SU, 0x80},
|
||||
/* 622 */ {SR, 0x81},
|
||||
/* 623 */ {SR, 0x83},
|
||||
/* 624 */ {SR, 0x85},
|
||||
/* 625 */ {SR, 0x87},
|
||||
/* 626 */ {SD, 0x89},
|
||||
/* 627 */ {SR, 0x8D},
|
||||
/* 628 */ {SD, 0x8F},
|
||||
/* 629 */ {SR, 0x93},
|
||||
/* 62A */ {SD, 0x95},
|
||||
/* 62B */ {SD, 0x99},
|
||||
/* 62C */ {SD, 0x9D},
|
||||
/* 62D */ {SD, 0xA1},
|
||||
/* 62E */ {SD, 0xA5},
|
||||
/* 62F */ {SR, 0xA9},
|
||||
/* 630 */ {SR, 0xAB},
|
||||
/* 631 */ {SR, 0xAD},
|
||||
/* 632 */ {SR, 0xAF},
|
||||
/* 633 */ {SD, 0xB1},
|
||||
/* 634 */ {SD, 0xB5},
|
||||
/* 635 */ {SD, 0xB9},
|
||||
/* 636 */ {SD, 0xBD},
|
||||
/* 637 */ {SD, 0xC1},
|
||||
/* 638 */ {SD, 0xC5},
|
||||
/* 639 */ {SD, 0xC9},
|
||||
/* 63A */ {SD, 0xCD},
|
||||
/* 63B */ {SU, 0x0},
|
||||
/* 63C */ {SU, 0x0},
|
||||
/* 63D */ {SU, 0x0},
|
||||
/* 63E */ {SU, 0x0},
|
||||
/* 63F */ {SU, 0x0},
|
||||
/* 640 */ {SC, 0x0},
|
||||
/* 641 */ {SD, 0xD1},
|
||||
/* 642 */ {SD, 0xD5},
|
||||
/* 643 */ {SD, 0xD9},
|
||||
/* 644 */ {SD, 0xDD},
|
||||
/* 645 */ {SD, 0xE1},
|
||||
/* 646 */ {SD, 0xE5},
|
||||
/* 647 */ {SD, 0xE9},
|
||||
/* 648 */ {SR, 0xED},
|
||||
/* 649 */ {SR, 0xEF},
|
||||
/* 64A */ {SD, 0xF1}};
|
||||
|
||||
/* ── CrossPoint extension: joining types outside U+0621–U+064A ──────────
|
||||
* Source: Unicode ArabicShaping.txt (joining type column). Ranges of
|
||||
* letters sharing one type are collapsed. Sorted ascending (binary
|
||||
* search). Anything not listed here or in shapetypes[] is SU. */
|
||||
static const struct {
|
||||
ucschar first, last;
|
||||
uchar type;
|
||||
} xjointypes[] = {
|
||||
{0x0620, 0x0620, SD}, /* kashmiri yeh */
|
||||
{0x066E, 0x066F, SD}, /* dotless beh/qaf */
|
||||
{0x0671, 0x0673, SR}, /* alef wasla + wavy-hamza alefs */
|
||||
{0x0675, 0x0677, SR}, /* high-hamza alef/waw */
|
||||
{0x0678, 0x0687, SD}, /* high-hamza yeh, beh group (ٹ ٺ ٻ … پ), hah group (چ ڇ) */
|
||||
{0x0688, 0x0699, SR}, /* dal group (ڈ ڊ ڌ …), reh group (ڑ ړ ژ …) */
|
||||
{0x069A, 0x06BF, SD}, /* seen/sad/feh/qaf/kaf/gaf/lam/noon groups (ک گ ں ھ …) */
|
||||
{0x06C0, 0x06C0, SR}, /* heh with yeh above */
|
||||
{0x06C1, 0x06C2, SD}, /* heh goal (ہ) */
|
||||
{0x06C3, 0x06CB, SR}, /* teh marbuta goal, waw group (ۆ ۇ ۋ …) */
|
||||
{0x06CC, 0x06CC, SD}, /* farsi yeh (ی) */
|
||||
{0x06CD, 0x06CD, SR}, /* yeh with tail */
|
||||
{0x06CE, 0x06CE, SD}, /* farsi yeh with V (Kurdish ێ) */
|
||||
{0x06CF, 0x06CF, SR}, /* waw with dot above */
|
||||
{0x06D0, 0x06D1, SD}, /* e (Pashto ې), yeh three dots below */
|
||||
{0x06D2, 0x06D3, SR}, /* yeh barree (ے ۓ) */
|
||||
{0x06D5, 0x06D5, SR}, /* ae (Kurdish ە) */
|
||||
{0x06EE, 0x06EF, SR}, /* dal/reh with inverted V */
|
||||
{0x06FA, 0x06FC, SD}, /* sheen/dad/ghain with dot below */
|
||||
{0x06FF, 0x06FF, SD}, /* heh with inverted V */
|
||||
{0x200C, 0x200C, SU}, /* ZWNJ — joining type U per ArabicShaping.txt */
|
||||
{0x200D, 0x200D, SC}, /* ZWJ — joining type C per ArabicShaping.txt */
|
||||
};
|
||||
|
||||
/* ── CrossPoint extension: presentation forms outside U+0621–U+064A ─────
|
||||
* Source: UnicodeData.txt, Arabic Presentation Forms-A (U+FB50–U+FBFF).
|
||||
* forms = number of consecutive presentation forms allocated for the
|
||||
* letter, always in the order isolated, final, initial, medial (matching
|
||||
* the SHAPE_* offsets below): 2 = isolated+final, 4 = all.
|
||||
* Letters absent here have no presentation forms and keep their base
|
||||
* codepoint. Sorted ascending (binary search). */
|
||||
static const struct {
|
||||
ucschar cp;
|
||||
ucschar isolated;
|
||||
uchar forms;
|
||||
} xshapeforms[] = {
|
||||
{0x0671, 0xFB50, 2}, /* alef wasla */
|
||||
{0x0679, 0xFB66, 4}, /* tteh (Urdu ٹ) */
|
||||
{0x067A, 0xFB5E, 4}, /* tteheh */
|
||||
{0x067B, 0xFB52, 4}, /* beeh (Sindhi ٻ) */
|
||||
{0x067E, 0xFB56, 4}, /* peh (Farsi پ) */
|
||||
{0x067F, 0xFB62, 4}, /* teheh */
|
||||
{0x0680, 0xFB5A, 4}, /* beheh (Sindhi ڀ) */
|
||||
{0x0683, 0xFB76, 4}, /* nyeh (Sindhi ڃ) */
|
||||
{0x0684, 0xFB72, 4}, /* dyeh (Sindhi ڄ) */
|
||||
{0x0686, 0xFB7A, 4}, /* tcheh (Farsi چ) */
|
||||
{0x0687, 0xFB7E, 4}, /* tcheheh (Sindhi ڇ) */
|
||||
{0x0688, 0xFB88, 2}, /* ddal (Urdu ڈ) */
|
||||
{0x068C, 0xFB84, 2}, /* dahal (Sindhi ڌ) */
|
||||
{0x068D, 0xFB82, 2}, /* ddahal (Sindhi ڍ) */
|
||||
{0x068E, 0xFB86, 2}, /* dul (Sindhi ڎ) */
|
||||
{0x0691, 0xFB8C, 2}, /* rreh (Urdu ڑ) */
|
||||
{0x0698, 0xFB8A, 2}, /* jeh (Farsi ژ) */
|
||||
{0x06A4, 0xFB6A, 4}, /* veh (Kurdish ڤ) */
|
||||
{0x06A6, 0xFB6E, 4}, /* peheh (Sindhi ڦ) */
|
||||
{0x06A9, 0xFB8E, 4}, /* keheh (Farsi/Urdu ک) */
|
||||
{0x06AD, 0xFBD3, 4}, /* ng */
|
||||
{0x06AF, 0xFB92, 4}, /* gaf (Farsi/Urdu گ) */
|
||||
{0x06B1, 0xFB9A, 4}, /* ngoeh (Sindhi ڱ) */
|
||||
{0x06B3, 0xFB96, 4}, /* gueh (Sindhi ڳ) */
|
||||
{0x06BA, 0xFB9E, 2}, /* noon ghunna (Urdu ں) — dual-joining but only
|
||||
isolated+final forms exist; initial/medial
|
||||
contexts keep the base codepoint */
|
||||
{0x06BB, 0xFBA0, 4}, /* rnoon (Sindhi ڻ) */
|
||||
{0x06BE, 0xFBAA, 4}, /* heh doachashmee (Urdu ھ) */
|
||||
{0x06C0, 0xFBA4, 2}, /* heh with yeh above */
|
||||
{0x06C1, 0xFBA6, 4}, /* heh goal (Urdu ہ) */
|
||||
{0x06C5, 0xFBE0, 2}, /* kirghiz oe */
|
||||
{0x06C6, 0xFBD9, 2}, /* oe (Kurdish ۆ) */
|
||||
{0x06C7, 0xFBD7, 2}, /* u (ۇ) */
|
||||
{0x06C8, 0xFBDB, 2}, /* yu */
|
||||
{0x06C9, 0xFBE2, 2}, /* kirghiz yu */
|
||||
{0x06CB, 0xFBDE, 2}, /* ve */
|
||||
{0x06CC, 0xFBFC, 4}, /* farsi yeh (Farsi/Urdu ی) */
|
||||
{0x06D0, 0xFBE4, 4}, /* e (Pashto ې) */
|
||||
{0x06D2, 0xFBAE, 2}, /* yeh barree (Urdu ے) */
|
||||
{0x06D3, 0xFBB0, 2}, /* yeh barree with hamza above (ۓ) */
|
||||
};
|
||||
|
||||
/* Contextual form offsets from the isolated form — identical ordering in
|
||||
Presentation Forms-A and -B, matching mintty's SFINAL/SINITIAL/SMEDIAL
|
||||
(+1/+2/+3) macros. */
|
||||
enum { SHAPE_ISOLATED = 0, SHAPE_FINAL = 1, SHAPE_INITIAL = 2, SHAPE_MEDIAL = 3 };
|
||||
|
||||
/* STYPE equivalent (mintty macro → function to add the extended table) */
|
||||
static uchar stype(ucschar c) {
|
||||
if (c >= SHAPE_FIRST && c <= SHAPE_LAST) return shapetypes[c - SHAPE_FIRST].type;
|
||||
|
||||
int i = -1, j = lengthof(xjointypes);
|
||||
while (j - i > 1) {
|
||||
int k = (i + j) / 2;
|
||||
if (c < xjointypes[k].first)
|
||||
j = k;
|
||||
else if (c > xjointypes[k].last)
|
||||
i = k;
|
||||
else
|
||||
return xjointypes[k].type;
|
||||
}
|
||||
return SU;
|
||||
}
|
||||
|
||||
/* SISOLATED/SFINAL/SINITIAL/SMEDIAL equivalent.
|
||||
form is one of the SHAPE_* offsets; returns c unchanged when the letter
|
||||
has no presentation form allocated for that context. */
|
||||
static ucschar shape_form(ucschar c, uchar form) {
|
||||
if (c >= SHAPE_FIRST && c <= SHAPE_LAST) {
|
||||
const uchar form_b = shapetypes[c - SHAPE_FIRST].form_b;
|
||||
return form_b ? 0xFE00 + form_b + form : c;
|
||||
}
|
||||
|
||||
int i = -1, j = lengthof(xshapeforms);
|
||||
while (j - i > 1) {
|
||||
int k = (i + j) / 2;
|
||||
if (c < xshapeforms[k].cp)
|
||||
j = k;
|
||||
else if (c > xshapeforms[k].cp)
|
||||
i = k;
|
||||
else
|
||||
return form < xshapeforms[k].forms ? xshapeforms[k].isolated + form : c;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
/* CrossPoint deviation 1: NSM marks (harakat) sit between letters in our
|
||||
array but are transparent for joining (Unicode joining type T). These
|
||||
helpers find the effective joining neighbour. */
|
||||
static int next_non_nsm(const bidi_char* line, int i, int count) {
|
||||
int j = i + 1;
|
||||
while (j < count && bidi_class(line[j].wc) == NSM) j++;
|
||||
return j;
|
||||
}
|
||||
|
||||
static int prev_non_nsm(const bidi_char* line, int i) {
|
||||
int j = i - 1;
|
||||
while (j >= 0 && bidi_class(line[j].wc) == NSM) j--;
|
||||
return j;
|
||||
}
|
||||
|
||||
/* The Main shaping function (mintty, structure preserved).
|
||||
*
|
||||
* line: visual-order buffer — must have been passed through do_bidi() first
|
||||
* to: output buffer for the shaped data
|
||||
* count: number of characters in line
|
||||
*/
|
||||
int do_shape(bidi_char* line, bidi_char* to, int count) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
to[i] = line[i];
|
||||
int tempShape = stype(line[i].wc);
|
||||
switch (tempShape) {
|
||||
case SR: { /* Right-Joining, i.e. has Isolated, Final */
|
||||
const int nx = next_non_nsm(line, i, count); /* deviation 1: was i + 1 */
|
||||
tempShape = (nx < count) ? stype(line[nx].wc) : SU;
|
||||
if (tempShape == SL || tempShape == SD || tempShape == SC)
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_FINAL);
|
||||
else
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED);
|
||||
break;
|
||||
}
|
||||
case SD: { /* Dual-Joining, i.e. has Isolated, Final, Initial, Medial */
|
||||
const int nx = next_non_nsm(line, i, count); /* deviation 1: was i + 1 */
|
||||
const int pv = prev_non_nsm(line, i); /* deviation 1: was i - 1 */
|
||||
|
||||
/* Make Ligatures */
|
||||
tempShape = (nx < count) ? stype(line[nx].wc) : SU;
|
||||
if (line[i].wc == 0x644) { /* Lam: the Alef variant is at pv (visually left) */
|
||||
int ligFlag = 0;
|
||||
switch (pv >= 0 ? line[pv].wc : 0) {
|
||||
case 0x622: /* Alef with Madda above → لآ */
|
||||
ligFlag = 1;
|
||||
to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEF6 : 0xFEF5;
|
||||
break;
|
||||
case 0x623: /* Alef with Hamza above → لأ */
|
||||
ligFlag = 1;
|
||||
to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEF8 : 0xFEF7;
|
||||
break;
|
||||
case 0x625: /* Alef with Hamza below → لإ */
|
||||
ligFlag = 1;
|
||||
to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEFA : 0xFEF9;
|
||||
break;
|
||||
case 0x627: /* Alef → لا */
|
||||
ligFlag = 1;
|
||||
to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEFC : 0xFEFB;
|
||||
break;
|
||||
}
|
||||
if (ligFlag) {
|
||||
to[pv].wc = LIGATURE_PLACEHOLDER; /* deviation 2: mintty writes 0x20 */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Arabic joining formatters: adapt forms (mintty) */
|
||||
const uchar joiners = line[i].joiners & 0xF;
|
||||
const uchar prevjoiners = line[i].joiners >> 4;
|
||||
if (prevjoiners == ZWNJ) {
|
||||
/* backward join blocked; initial only if the visually-right
|
||||
(logically next) neighbour joins, else isolated */
|
||||
tempShape = (pv >= 0) ? stype(line[pv].wc) : SU;
|
||||
if (tempShape == SR || tempShape == SD || tempShape == SC)
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_INITIAL);
|
||||
else
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED);
|
||||
} else if (prevjoiners == (ZWJ | ZWNJ)) {
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_MEDIAL);
|
||||
} else if (prevjoiners == ZWJ) {
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_FINAL);
|
||||
} else if (joiners & ZWNJ) {
|
||||
/* forward join blocked; final only if the visually-left
|
||||
(logically previous) neighbour joins, else isolated —
|
||||
tempShape still holds stype(nx) from above */
|
||||
if (tempShape == SL || tempShape == SD || tempShape == SC)
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_FINAL);
|
||||
else
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED);
|
||||
} else if (tempShape == SL || tempShape == SD || tempShape == SC) {
|
||||
/* visually-right neighbour joins → final, or medial if the
|
||||
visually-left neighbour joins too */
|
||||
tempShape = (pv >= 0) ? stype(line[pv].wc) : SU;
|
||||
if (tempShape == SR || tempShape == SD || tempShape == SC)
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_MEDIAL);
|
||||
else
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_FINAL);
|
||||
} else {
|
||||
/* visually-right neighbour doesn't join → isolated, or initial
|
||||
if the visually-left neighbour joins */
|
||||
tempShape = (pv >= 0) ? stype(line[pv].wc) : SU;
|
||||
if (tempShape == SR || tempShape == SD || tempShape == SC)
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_INITIAL);
|
||||
else
|
||||
to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
/* SU, SL, SC and non-Arabic characters pass through unchanged */
|
||||
break;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════
|
||||
* Directional Status Stack
|
||||
* (replaces GCC nested functions — ESP32C3 has no executable stack)
|
||||
|
||||
+52
-5
@@ -5,8 +5,10 @@
|
||||
* minibidi.h — standalone header for ESP32C3 BiDi calculations
|
||||
*
|
||||
* Derived from [mintty](https://github.com/mintty/mintty/) (Thomas Wolff, MIT licence).
|
||||
* Stripped of: Arabic shaping, box-drawing mirror, terminal dependencies,
|
||||
* GCC nested functions, VLAs, and non-Hebrew/English Unicode data.
|
||||
* Includes: UAX#9 bidi (do_bidi) and Arabic contextual shaping (do_shape),
|
||||
* both ported from mintty src/minibidi.c (Ahmad Khalifa, Thomas Wolff).
|
||||
* Stripped of: box-drawing mirror, terminal dependencies, GCC nested
|
||||
* functions, VLAs, and Unicode data for scripts CrossPoint doesn't render.
|
||||
*/
|
||||
|
||||
#include <stdbool.h>
|
||||
@@ -34,13 +36,18 @@ typedef uint32_t ucschar; /* Unicode codepoint; BMP-only content fits uint16_t
|
||||
#define BIDI_MAX_LINE 128
|
||||
|
||||
/* ── bidi_char ───────────────────────────────────────────────────────── */
|
||||
/* origwc: the codepoint as it came from the epub text stream
|
||||
wc: working codepoint (may be replaced by mirrored form after do_bidi)
|
||||
index: original logical position, so the caller can reorder glyphs */
|
||||
/* origwc: the codepoint as it came from the epub text stream
|
||||
wc: working codepoint (may be replaced by mirrored form after
|
||||
do_bidi, or by an Arabic contextual form after do_shape)
|
||||
index: original logical position, so the caller can reorder glyphs
|
||||
joiners: ZWJ/ZWNJ context for Arabic shaping, mintty layout:
|
||||
low nibble = joiners that logically FOLLOW this character,
|
||||
high nibble = joiners that logically PRECEDE this character */
|
||||
typedef struct {
|
||||
ucschar origwc;
|
||||
ucschar wc;
|
||||
uint16_t index;
|
||||
uint8_t joiners;
|
||||
} bidi_char;
|
||||
|
||||
/* ── Bidi character classes (UAX #9) ────────────────────────────────── */
|
||||
@@ -71,6 +78,20 @@ enum {
|
||||
PDI, /* Pop Directional Isolate */
|
||||
};
|
||||
|
||||
/* ── Arabic joining formatter flags (bidi_char.joiners nibbles) ─────── */
|
||||
/* Values match mintty's minibidi.h: ZWNJ 0x01, ZWJ 0x02. do_shape()
|
||||
compares nibble values directly, so these must not be changed. */
|
||||
enum {
|
||||
ZWNJ = 0x01, /* U+200C ZERO WIDTH NON-JOINER */
|
||||
ZWJ = 0x02, /* U+200D ZERO WIDTH JOINER */
|
||||
};
|
||||
|
||||
/* Sentinel written by do_shape() over the Alef absorbed into a Lam-Alef
|
||||
ligature. Callers must filter it out when emitting shaped text.
|
||||
(Upstream mintty writes a space instead — a terminal must keep the cell;
|
||||
a proportional-text renderer must drop the character entirely.) */
|
||||
#define LIGATURE_PLACEHOLDER 0xFFFFu
|
||||
|
||||
/* ── Public API ──────────────────────────────────────────────────────── */
|
||||
|
||||
/*
|
||||
@@ -94,6 +115,32 @@ bool is_rtl_class(uchar bc);
|
||||
*/
|
||||
ucschar mirror(ucschar ch);
|
||||
|
||||
/*
|
||||
* do_shape(line, to, count)
|
||||
*
|
||||
* Applies Arabic contextual shaping (and Lam-Alef ligation) to
|
||||
* `line[0..count-1]`, writing the result to `to[0..count-1]`.
|
||||
*
|
||||
* MUST be called AFTER do_bidi(): the algorithm resolves joining from
|
||||
* VISUAL adjacency (line[i-1] is the visually-left neighbour, line[i+1]
|
||||
* the visually-right one), exactly like upstream mintty.
|
||||
*
|
||||
* line: visual-order input; the joiners field must be populated by the
|
||||
* caller (in logical order, before do_bidi) for ZWJ/ZWNJ support
|
||||
* to: output buffer, same size as line; non-Arabic entries are copied
|
||||
* through unchanged. An Alef absorbed by a Lam-Alef ligature is
|
||||
* replaced with LIGATURE_PLACEHOLDER — filter it on emission.
|
||||
* count: number of characters (≤ BIDI_MAX_LINE)
|
||||
*
|
||||
* Returns 1.
|
||||
*
|
||||
* Ported from mintty src/minibidi.c (Ahmad Khalifa, Thomas Wolff,
|
||||
* MIT licence), https://github.com/mintty/mintty — with CrossPoint
|
||||
* extensions for Perso-Arabic letters and in-stream diacritics, see
|
||||
* minibidi.c for details.
|
||||
*/
|
||||
int do_shape(bidi_char* line, bidi_char* to, int count);
|
||||
|
||||
/*
|
||||
* do_bidi(autodir, paragraphLevel, line, count)
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user