diff --git a/docs/file-formats.md b/docs/file-formats.md index 4289f7e1..b66e05a6 100644 --- a/docs/file-formats.md +++ b/docs/file-formats.md @@ -90,13 +90,19 @@ if (parsedSize != fileSize) { ## `section.bin` -### Version 29 +### Version 30 Each file in `sections/*.bin` stores one laid-out spine section. The header is also the cache-busting key: if any layout-affecting setting differs from the current reader settings, the section is discarded and rebuilt. -Version 29 includes: +Version 30 is binary-identical to version 29. The version was bumped because +Arabic contextual shaping changed text measurement (`getTextAdvanceX` now +measures the shaped visual text), so word positions cached by v29 no longer +match what `drawText` renders. + +Version 28 introduced serialized word style bits for underline, strikethrough, +superscript, and subscript. The format also includes: - cache-busting fields for paragraph alignment, hyphenation, embedded CSS, image rendering mode, and Focus Reading @@ -119,7 +125,7 @@ import std.mem; import std.string; import std.core; -#define EXPECTED_VERSION 29 +#define EXPECTED_VERSION 30 #define MAX_STRING_LENGTH 65535 #define FOOTNOTE_NUMBER_LEN 32 #define FOOTNOTE_HREF_LEN 96 diff --git a/lib/EpdFont/EpdFont.cpp b/lib/EpdFont/EpdFont.cpp index 6055e845..d54a4f76 100644 --- a/lib/EpdFont/EpdFont.cpp +++ b/lib/EpdFont/EpdFont.cpp @@ -44,16 +44,17 @@ void EpdFont::getTextBounds(const char* string, const int startX, const int star continue; } - const int raiseBy = isCombining ? combiningMark::raiseAboveBase(glyph->top, glyph->height, lastBaseTop) : 0; + const combiningMark::Anchor anchor = combiningMark::anchorFor(cp); + const int raiseBy = isCombining ? combiningMark::raiseAboveBase(anchor, glyph->top, glyph->height, lastBaseTop) : 0; if (!isCombining && prevCp != 0) { const auto kernFP = getKerning(prevCp, cp); // 4.4 fixed-point kern lastBaseX += fp4::toPixel(prevAdvanceFP + kernFP); } - const int glyphBaseX = - isCombining ? combiningMark::centerOver(lastBaseX, lastBaseLeft, lastBaseWidth, glyph->left, glyph->width) - : lastBaseX; + const int glyphBaseX = isCombining ? combiningMark::anchorOver(anchor, lastBaseX, lastBaseLeft, lastBaseWidth, + glyph->left, glyph->width) + : lastBaseX; const int glyphBaseY = startY - raiseBy; *minX = std::min(*minX, glyphBaseX + glyph->left); diff --git a/lib/EpdFont/EpdFontData.h b/lib/EpdFont/EpdFontData.h index cb1d5250..498d0926 100644 --- a/lib/EpdFont/EpdFontData.h +++ b/lib/EpdFont/EpdFontData.h @@ -36,24 +36,71 @@ namespace combiningMark { constexpr int MIN_GAP_PX = 1; -/// Compute the cursor-X at which to render a combining mark so its bitmap -/// is visually centered over the base glyph's bitmap. -constexpr int centerOver(int baseCursorPos, int baseLeft, int baseWidth, int markLeft, int markWidth) { - return baseCursorPos + baseLeft + baseWidth / 2 - markWidth / 2 - markLeft; +/// Placement of a mark relative to its base glyph. The default heuristic — +/// centered over the base, raised clear of its top — suits Latin diacritics +/// and Arabic harakat, but misplaces the Hebrew niqqud whose identity depends +/// on position: dagesh sits inside the letter body, the shin/sin dots +/// distinguish the letter by sitting over its right/left arm, and holam hangs +/// over the left corner. "Native" anchors keep the glyph's font-designed +/// height (which may overlap the base) instead of raising it. +enum class Anchor : uint8_t { + CenterRaised, ///< centered over the base, lifted above its top (default) + CenterNative, ///< centered over the base at font-native height + RightNative, ///< right edges aligned, font-native height + LeftNative, ///< left edges aligned, font-native height +}; + +constexpr Anchor anchorFor(const uint32_t cp) { + switch (cp) { + case 0x05BC: // dagesh / mapiq / shuruk dot: inside the letter body + case 0x05BA: // holam haser for vav: straight above the vav stem + return Anchor::CenterNative; + case 0x05C1: // shin dot: over the letter's right arm + return Anchor::RightNative; + case 0x05B9: // holam: above the letter's left corner + case 0x05C2: // sin dot: over the letter's left arm + return Anchor::LeftNative; + default: + return Anchor::CenterRaised; + } } -/// Rotated-90CW variant of centerOver. In the rotated coordinate system +/// Horizontal offset from the base bitmap's left edge to the mark bitmap's +/// left edge for a given anchor. +constexpr int anchorShift(const Anchor anchor, const int baseWidth, const int markWidth) { + switch (anchor) { + case Anchor::LeftNative: + return 0; + case Anchor::RightNative: + return baseWidth - markWidth; + default: + return baseWidth / 2 - markWidth / 2; + } +} + +/// Compute the cursor-X at which to render a combining mark so its bitmap +/// lands at its anchor position over the base glyph's bitmap. +constexpr int anchorOver(const Anchor anchor, const int baseCursorPos, const int baseLeft, const int baseWidth, + const int markLeft, const int markWidth) { + return baseCursorPos + baseLeft + anchorShift(anchor, baseWidth, markWidth) - markLeft; +} + +/// Rotated-90CW variant of anchorOver. In the rotated coordinate system /// renderCharImpl uses (cursorY - left) instead of (cursorX + left), so /// every left/width term inverts sign. -constexpr int centerOverRotated90CW(int baseCursorPos, int baseLeft, int baseWidth, int markLeft, int markWidth) { - return baseCursorPos - baseLeft - baseWidth / 2 + markWidth / 2 + markLeft; +constexpr int anchorOverRotated90CW(const Anchor anchor, const int baseCursorPos, const int baseLeft, + const int baseWidth, const int markLeft, const int markWidth) { + return baseCursorPos - baseLeft - anchorShift(anchor, baseWidth, markWidth) + markLeft; } /// For combining marks that sit entirely above the baseline, compute how many /// pixels to raise the mark so there is at least MIN_GAP_PX between its bottom /// edge and the top of the base glyph. Returns 0 for marks that extend to or -/// below the baseline (e.g. cedilla, dot-below, ogonek). -constexpr int raiseAboveBase(int markTop, int markHeight, int baseTop) { +/// below the baseline (e.g. cedilla, dot-below, ogonek) and for anchors that +/// keep the font-native height (dagesh must stay inside the letter, the +/// shin/sin dots touch its arms). +constexpr int raiseAboveBase(const Anchor anchor, const int markTop, const int markHeight, const int baseTop) { + if (anchor != Anchor::CenterRaised) return 0; if (markTop - markHeight <= 0) return 0; const int gap = markTop - markHeight - baseTop; return (gap < MIN_GAP_PX) ? (MIN_GAP_PX - gap) : 0; @@ -61,6 +108,20 @@ constexpr int raiseAboveBase(int markTop, int markHeight, int baseTop) { } // namespace combiningMark +/// GCC/Clang (the ESP32 firmware toolchain) pack structs with __attribute__((packed)). +/// MSVC (host unit tests) has no equivalent attribute and instead needs a #pragma pack +/// region achieving the same 1-byte alignment. These macros keep the on-disk font layout +/// identical across both toolchains. +#if defined(_MSC_VER) +#define EPD_PACKED_BEGIN __pragma(pack(push, 1)) +#define EPD_PACKED_END __pragma(pack(pop)) +#define EPD_PACKED_ATTR +#else +#define EPD_PACKED_BEGIN +#define EPD_PACKED_END +#define EPD_PACKED_ATTR __attribute__((packed)) +#endif + /// Fixed-point conventions used by EpdGlyph and EpdFontData: /// advanceX: 12.4 unsigned fixed-point in uint16_t (use fp4::toPixel) /// kernMatrix: 4.4 signed fixed-point in int8_t (use fp4::toPixel) @@ -95,17 +156,21 @@ typedef struct { /// Maps a codepoint to a kerning class ID, sorted by codepoint for binary search. /// Class IDs are 1-based; codepoints not in the table have implicit class 0 (no kerning). +EPD_PACKED_BEGIN typedef struct { uint16_t codepoint; ///< Unicode codepoint uint8_t classId; ///< 1-based kerning class ID -} __attribute__((packed)) EpdKernClassEntry; +} EPD_PACKED_ATTR EpdKernClassEntry; +EPD_PACKED_END /// Ligature substitution for a specific glyph pair, sorted by `pair` for binary search. /// `pair` encodes (leftCodepoint << 16 | rightCodepoint) for single-key lookup. +EPD_PACKED_BEGIN typedef struct { uint32_t pair; ///< Packed codepoint pair (left << 16 | right) uint32_t ligatureCp; ///< Codepoint of the replacement ligature glyph -} __attribute__((packed)) EpdLigaturePair; +} EPD_PACKED_ATTR EpdLigaturePair; +EPD_PACKED_END /// Data stored for FONT AS A WHOLE typedef struct { diff --git a/lib/EpdFont/SdCardFont.cpp b/lib/EpdFont/SdCardFont.cpp index 1609ab3a..90da9635 100644 --- a/lib/EpdFont/SdCardFont.cpp +++ b/lib/EpdFont/SdCardFont.cpp @@ -1195,7 +1195,8 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun } template -int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask) { +int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask, + const char* extraText) { if (!loaded_) return -1; styleMask = resolveStyleMask(styleMask); if (styleMask == 0) return 0; @@ -1215,6 +1216,9 @@ int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, for (auto it = begin; it != end && !hitCap; ++it) { hitCap = collectUniqueCodepoints(asCStr(*it), codepoints, cpCount, MAX_UNIQUE_CODEPOINTS); } + if (extraText && !hitCap) { + hitCap = collectUniqueCodepoints(extraText, codepoints, cpCount, MAX_UNIQUE_CODEPOINTS); + } if (includeSpace && std::none_of(codepoints, codepoints + cpCount, [](uint32_t c) { return c == ' '; })) codepoints[cpCount++] = ' '; @@ -1232,12 +1236,13 @@ int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, return totalMissed; } -int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask) { - return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask); +int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask, const char* extraText) { + return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask, extraText); } -int SdCardFont::buildAdvanceTable(const std::vector& words, bool includeHyphen, uint8_t styleMask) { - return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask); +int SdCardFont::buildAdvanceTable(const std::vector& words, bool includeHyphen, uint8_t styleMask, + const char* extraText) { + return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask, extraText); } // --- Stats --- diff --git a/lib/EpdFont/SdCardFont.h b/lib/EpdFont/SdCardFont.h index 443fee68..422b70c9 100644 --- a/lib/EpdFont/SdCardFont.h +++ b/lib/EpdFont/SdCardFont.h @@ -47,9 +47,12 @@ class SdCardFont { // Build a compact advance-only table for layout measurement. // Extracts ALL unique codepoints from words (no MAX_PAGE_GLYPHS cap), // batch-reads advanceX from SD, stores in a sorted per-style table. + // extraText: optional additional codepoints to warm in the same SD pass + // (e.g. shaped Arabic presentation forms the measurement path will look up). // Returns number of codepoints not found in font coverage. - int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F); - int buildAdvanceTable(const std::vector& words, bool includeHyphen, uint8_t styleMask = 0x0F); + int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F, const char* extraText = nullptr); + int buildAdvanceTable(const std::vector& words, bool includeHyphen, uint8_t styleMask = 0x0F, + const char* extraText = nullptr); // Look up advanceX for a codepoint from the advance table. // Returns the 12.4 fixed-point advance, or 0 if not found. @@ -140,11 +143,13 @@ class SdCardFont { // Full intervals loaded from file (kept in RAM for codepoint lookup) EpdUnicodeInterval* fullIntervals = nullptr; + EPD_PACKED_BEGIN struct BmpInterval16 { uint16_t first; uint16_t last; uint16_t offset; - } __attribute__((packed)); + } EPD_PACKED_ATTR; + EPD_PACKED_END static_assert(sizeof(BmpInterval16) == 6, "BmpInterval16 must remain compact"); BmpInterval16* bmpIntervals = nullptr; bool intervalsAreBmp16 = false; @@ -249,7 +254,8 @@ class SdCardFont { int32_t findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint) const; int fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask); template - int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask); + int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask, + const char* extraText = nullptr); int prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint32_t cpCount, bool metadataOnly); // Global helpers diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index 663d086b..56daeed0 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -11,9 +11,13 @@ #include "parsers/ChapterHtmlSlimParser.h" namespace { +// v28: text decoration bits now include line-through in serialized wordStyles. // v29: TextBlock word data stored as one flat arena (offset table + NUL-terminated // text blob) instead of length-prefixed strings and per-field arrays. -constexpr uint8_t SECTION_FILE_VERSION = 29; +// v30: Arabic shaping changed both drawing and measurement (getTextAdvanceX now +// measures the shaped visual text); cached word positions from v29 no longer +// match what drawText renders. +constexpr uint8_t SECTION_FILE_VERSION = 30; // Written into the version field while a build is in progress; patched to // SECTION_FILE_VERSION only when the build is finalized. An abandoned / // crash-interrupted .bin therefore carries version 0, which loadSectionFile rejects diff --git a/lib/GfxRenderer/GfxRenderer.cpp b/lib/GfxRenderer/GfxRenderer.cpp index 6fcc21c6..e63ad51a 100644 --- a/lib/GfxRenderer/GfxRenderer.cpp +++ b/lib/GfxRenderer/GfxRenderer.cpp @@ -25,6 +25,36 @@ uint8_t resolveSdCardStyle(const SdCardFont& font, const EpdFontFamily::Style st namespace { const char* resolveVisualText(const char* text, std::string& visualBuffer, BidiUtils::BidiBaseDir baseDir); + +// Appends the shaped visual form of every RTL token in `text` to `shapedOut`. +// getTextAdvanceX() measures the bidi-reordered, Arabic-shaped codepoint stream, +// so the SD advance table must be warmed with the presentation forms as well as +// the logical codepoints — otherwise every RTL word measurement misses the fast +// path and falls through to onGlyphMiss(), which opens the .cpfont and reads +// glyph metadata + bitmap into the 8-slot overflow ring, once per glyph. +// Tokens without RTL lead bytes (0xD6-0xDB) are skipped with a byte scan, so +// pure-LTR text pays almost nothing. +void appendShapedRtlTokens(const char* text, std::string& shapedOut) { + const auto isBreak = [](const char c) { return c == ' ' || c == '\n' || c == '\r' || c == '\t'; }; + std::string token; + std::string visual; + const char* p = text; + while (*p) { + while (*p && isBreak(*p)) ++p; + const char* start = p; + bool hasRtlBytes = false; + while (*p && !isBreak(*p)) { + const auto b = static_cast(*p); + hasRtlBytes = hasRtlBytes || (b >= 0xD6 && b <= 0xDB); + ++p; + } + if (!hasRtlBytes) continue; + token.assign(start, p - start); + if (BidiUtils::applyBidiVisual(token.c_str(), visual, static_cast(BidiUtils::BidiBaseDir::AUTO))) { + shapedOut += visual; + } + } +} } // namespace const uint8_t* GfxRenderer::getGlyphBitmap(const EpdFontData* fontData, const EpdGlyph* glyph) const { @@ -58,7 +88,9 @@ const uint8_t* GfxRenderer::getGlyphBitmap(const EpdFontData* fontData, const Ep void GfxRenderer::ensureSdCardFontReady(int fontId, const char* utf8Text, uint8_t styleMask) const { auto it = sdCardFonts_.find(fontId); if (it != sdCardFonts_.end()) { - int missed = it->second->buildAdvanceTable(utf8Text, styleMask); + std::string shaped; + appendShapedRtlTokens(utf8Text, shaped); + int missed = it->second->buildAdvanceTable(utf8Text, styleMask, shaped.empty() ? nullptr : shaped.c_str()); if (missed > 0) { LOG_DBG("GFX", "ensureSdCardFontReady: %d glyph(s) not found", missed); } @@ -72,7 +104,12 @@ void GfxRenderer::ensureSdCardFontReady(int fontId, const std::vectorsecond->buildAdvanceTable(words, includeHyphen, styleMask); + std::string shaped; + for (const auto& w : words) { + appendShapedRtlTokens(w.c_str(), shaped); + } + int missed = + it->second->buildAdvanceTable(words, includeHyphen, styleMask, shaped.empty() ? nullptr : shaped.c_str()); if (missed > 0) { LOG_DBG("GFX", "ensureSdCardFontReady: %d glyph(s) not found", missed); } @@ -460,18 +497,21 @@ void GfxRenderer::drawText(const int fontId, const int x, const int y, const cha uint32_t cp; uint32_t prevCp = 0; while ((cp = utf8NextCodepoint(reinterpret_cast(&textCursor)))) { - // Skip Hebrew Niqqud (vowel marks) - // Temporary: avoid adding Niqqud to built-in fonts. Remove when custom fonts are supported. - if (cp >= 0x0591 && cp <= 0x05C7) { - continue; - } - - if (utf8IsCombiningMark(cp)) { + // RTL vowel marks (Hebrew niqqud, Arabic harakat) ride the combining-mark + // path: zero-advance overlays on the preceding base glyph (applyBidiVisual + // emits base-then-marks per UAX#9 L3). anchorFor pins position-sensitive + // niqqud (dagesh, shin/sin dots, holam) to their spot on the base; other + // marks stay centered, raised above the base or (kasra) at their + // font-native position. Fonts without their glyphs — the built-ins — miss + // the getGlyph lookup and skip them, as before. + if (utf8IsCombiningMark(cp) || BidiUtils::isTransparentMark(cp)) { const EpdGlyph* combiningGlyph = font.getGlyph(cp, style); if (!combiningGlyph) continue; - const int raiseBy = combiningMark::raiseAboveBase(combiningGlyph->top, combiningGlyph->height, lastBaseTop); - const int combiningX = combiningMark::centerOver(lastBaseX, lastBaseLeft, lastBaseWidth, combiningGlyph->left, - combiningGlyph->width); + const auto anchor = combiningMark::anchorFor(cp); + const int raiseBy = + combiningMark::raiseAboveBase(anchor, combiningGlyph->top, combiningGlyph->height, lastBaseTop); + const int combiningX = combiningMark::anchorOver(anchor, lastBaseX, lastBaseLeft, lastBaseWidth, + combiningGlyph->left, combiningGlyph->width); renderCharImpl(*this, renderMode, font, cp, combiningX, yPos - raiseBy, black, style); continue; } @@ -1657,6 +1697,15 @@ int GfxRenderer::getKerning(const int fontId, const uint32_t leftCp, const uint3 } int GfxRenderer::getTextAdvanceX(const int fontId, const char* text, EpdFontFamily::Style style) const { + // Measure the exact codepoint stream drawText renders: bidi-reordered and + // Arabic-shaped (contextual presentation forms, Lam-Alef collapse). + // Measuring the raw logical text counts the Alef a ligature absorbs and + // uses base-letter advances instead of presentation-form advances, so RTL + // lines come out wider than they draw — uneven word gaps and a ragged + // right margin. + std::string visual; + text = resolveVisualText(text, visual, BidiUtils::BidiBaseDir::AUTO); + // Advance table fast-path for SD card fonts during layout. // No kerning/ligature lookup — consistent with previous metadataOnly behavior // where kern/lig data was not loaded. @@ -1672,6 +1721,10 @@ int GfxRenderer::getTextAdvanceX(const int fontId, const char* text, EpdFontFami } const auto& font = fontIt->second; while (uint32_t cp = utf8NextCodepoint(reinterpret_cast(&text))) { + // RTL vowel marks (niqqud/harakat) are zero-advance overlays in drawText — no width. + if (BidiUtils::isTransparentMark(cp)) { + continue; + } int32_t advFP = sdIt->second->getAdvance(cp, styleIdx); if (advFP == 0 && !utf8IsCombiningMark(cp)) { const EpdGlyph* glyph = font.getGlyph(cp, style); @@ -1694,6 +1747,10 @@ int GfxRenderer::getTextAdvanceX(const int fontId, const char* text, EpdFontFami int32_t prevAdvanceFP = 0; // 12.4 fixed-point: prev glyph's advance + next kern for snap const auto& font = fontIt->second; while ((cp = utf8NextCodepoint(reinterpret_cast(&text)))) { + // RTL vowel marks (niqqud/harakat) are zero-advance overlays in drawText — no width. + if (BidiUtils::isTransparentMark(cp)) { + continue; + } if (utf8IsCombiningMark(cp)) { continue; } @@ -1770,18 +1827,21 @@ void GfxRenderer::drawTextRotated90CW(const int fontId, const int x, const int y uint32_t cp; uint32_t prevCp = 0; while ((cp = utf8NextCodepoint(reinterpret_cast(&text)))) { - // Skip Hebrew Niqqud (vowel marks) - // Temporary: avoid adding Niqqud to built-in fonts. Remove when custom fonts are supported. - if (cp >= 0x0591 && cp <= 0x05C7) { - continue; - } - - if (utf8IsCombiningMark(cp)) { + // RTL vowel marks (Hebrew niqqud, Arabic harakat) ride the combining-mark + // path: zero-advance overlays on the preceding base glyph (applyBidiVisual + // emits base-then-marks per UAX#9 L3). anchorFor pins position-sensitive + // niqqud (dagesh, shin/sin dots, holam) to their spot on the base; other + // marks stay centered, raised above the base or (kasra) at their + // font-native position. Fonts without their glyphs — the built-ins — miss + // the getGlyph lookup and skip them, as before. + if (utf8IsCombiningMark(cp) || BidiUtils::isTransparentMark(cp)) { const EpdGlyph* combiningGlyph = font.getGlyph(cp, style); if (!combiningGlyph) continue; - const int raiseBy = combiningMark::raiseAboveBase(combiningGlyph->top, combiningGlyph->height, lastBaseTop); + const auto anchor = combiningMark::anchorFor(cp); + const int raiseBy = + combiningMark::raiseAboveBase(anchor, combiningGlyph->top, combiningGlyph->height, lastBaseTop); const int combiningX = x - raiseBy; - const int combiningY = combiningMark::centerOverRotated90CW(lastBaseY, lastBaseLeft, lastBaseWidth, + const int combiningY = combiningMark::anchorOverRotated90CW(anchor, lastBaseY, lastBaseLeft, lastBaseWidth, combiningGlyph->left, combiningGlyph->width); renderCharImpl(*this, renderMode, font, cp, combiningX, combiningY, black, style); continue; diff --git a/lib/MiniBidi/BidiUtils.cpp b/lib/MiniBidi/BidiUtils.cpp index 07f061f8..168063f8 100644 --- a/lib/MiniBidi/BidiUtils.cpp +++ b/lib/MiniBidi/BidiUtils.cpp @@ -11,6 +11,13 @@ extern "C" { #include #include +#include + +// Guards the static bidi_char buffers in applyBidiVisual() and +// computeVisualWordOrder(). The bidi+shaping pipeline is not reentrant; +// this mutex serialises access so multi-core callers don't corrupt each +// other's intermediate state. +static std::mutex bidiMutex; namespace { @@ -68,11 +75,25 @@ int detectParagraphLevel(const char* utf8, const int fallbackLevel, const int ma return fallbackLevel & 1; } +bool isTransparentMark(const uint32_t cp) { + // RTL-script combining marks: Hebrew niqqud/cantillation and Arabic + // harakat/Quranic annotation. Transparent for Arabic joining (do_shape + // skips them), zero-advance for measurement, and rendered as overlays on + // the preceding base glyph when the active font carries their glyphs. + // The cp >= 0x0591 guard keeps Latin combining marks (U+0300-U+036F, also + // NSM) on their existing utf8IsCombiningMark() rendering path. + return cp >= 0x0591 && bidi_class(cp) == NSM; +} + bool applyBidiVisual(const char* utf8, std::string& out, int paragraphLevel) { if (!utf8 || !*utf8) return false; + const std::lock_guard lock(bidiMutex); static bidi_char line[BIDI_MAX_LINE]; + static bidi_char shaped[BIDI_MAX_LINE]; int count = 0; + int lastBase = -1; // last non-formatter character (mintty's ibase) + uint8_t pendingJoiners = 0; // ZWJ/ZWNJ seen since lastBase auto* p = reinterpret_cast(utf8); while (*p) { if (count >= BIDI_MAX_LINE) { @@ -84,18 +105,66 @@ bool applyBidiVisual(const char* utf8, std::string& out, int paragraphLevel) { if (!cp || cp == REPLACEMENT_GLYPH) break; line[count].origwc = line[count].wc = cp; line[count].index = static_cast(count); + line[count].joiners = 0; + + // Flag Arabic joining formatters mintty-style (termline.c): the ZWJ/ZWNJ + // goes into the low nibble of the character it follows and the high + // nibble of the character it precedes. Flags are assigned in logical + // order here; do_shape() reads them after reordering. + if (cp == 0x200C || cp == 0x200D) { + const uint8_t joiner = (cp == 0x200D) ? ZWJ : ZWNJ; + if (lastBase >= 0) line[lastBase].joiners |= joiner; + pendingJoiners |= joiner; + } else { + line[count].joiners = pendingJoiners << 4; + pendingJoiners = 0; + lastBase = count; + } count++; } if (!count) return false; const bool autodir = (paragraphLevel < 0); const int level = autodir ? 0 : (paragraphLevel & 1); + + // Order matters (mintty does the same): do_bidi() first to obtain visual + // order, then do_shape() — contextual forms are resolved from *visual* + // adjacency, and shaping presentation forms must never be reordered. do_bidi(autodir, level, line, count); + do_shape(line, shaped, count); out.clear(); out.reserve(std::strlen(utf8)); + // Lam-Alef collapse sentinel and zero-width joining formatters have done + // their job during shaping and have no glyphs to render. + const auto filtered = [](const uint32_t cp) { return cp == LIGATURE_PLACEHOLDER || cp == 0x200C || cp == 0x200D; }; for (int i = 0; i < count; i++) { - utf8AppendCodepoint(line[i].wc, out); + const uint32_t cp = shaped[i].wc; + if (filtered(cp)) continue; + if (!isTransparentMark(cp)) { + utf8AppendCodepoint(cp, out); + continue; + } + // UAX#9 rule L3: reversing an RTL run leaves combining marks *before* + // their base character. The renderer overlays a mark on the most + // recently drawn glyph, so emit the base first, then its marks in + // logical order. `index` is the original logical position: a base + // following its marks with a *lower* index means the run was reversed. + int j = i; // [i, j) = the run of marks (and filtered entries) + while (j < count && (filtered(shaped[j].wc) || isTransparentMark(shaped[j].wc))) j++; + if (j < count && shaped[j].index < shaped[i].index) { + utf8AppendCodepoint(shaped[j].wc, out); + for (int k = j - 1; k >= i; k--) { + if (isTransparentMark(shaped[k].wc)) utf8AppendCodepoint(shaped[k].wc, out); + } + i = j; // base already emitted + } else { + // Unreversed (or trailing, base-less) marks already follow their base. + for (int k = i; k < j; k++) { + if (isTransparentMark(shaped[k].wc)) utf8AppendCodepoint(shaped[k].wc, out); + } + i = j - 1; + } } return true; } @@ -105,6 +174,7 @@ bool computeVisualWordOrder(const std::vector& words, bool paragrap visualOrder.clear(); const size_t nWords = words.size(); if (nWords <= 1 || nWords > BIDI_MAX_LINE) return false; + const std::lock_guard lock(bidiMutex); static bidi_char line[BIDI_MAX_LINE]; int count = 0; @@ -121,6 +191,7 @@ bool computeVisualWordOrder(const std::vector& words, bool paragrap if (!cp || cp == REPLACEMENT_GLYPH) break; line[count].origwc = line[count].wc = cp; line[count].index = static_cast(w); + line[count].joiners = 0; count++; } @@ -131,6 +202,7 @@ bool computeVisualWordOrder(const std::vector& words, bool paragrap } line[count].origwc = line[count].wc = ' '; line[count].index = static_cast(nWords); + line[count].joiners = 0; count++; } } diff --git a/lib/MiniBidi/BidiUtils.h b/lib/MiniBidi/BidiUtils.h index c36c9a36..466375c8 100644 --- a/lib/MiniBidi/BidiUtils.h +++ b/lib/MiniBidi/BidiUtils.h @@ -13,6 +13,14 @@ bool startsWithRtl(const char* utf8, int maxStrongChars = RTL_PARAGRAPH_PROBE_DE int detectParagraphLevel(const char* utf8, int fallbackLevel = 0, int maxStrongChars = 64); +// True for RTL-script non-spacing marks (Hebrew niqqud/cantillation, Arabic +// harakat and Quranic annotation): zero-width for measurement, transparent for +// Arabic joining, and rendered as overlays on the preceding base glyph when +// the active font carries their glyphs (SD fonts; built-in fonts don't). +// Latin combining marks (U+0300-U+036F) intentionally return false — they are +// handled by the utf8IsCombiningMark() rendering path. +bool isTransparentMark(uint32_t cp); + // paragraphLevel: -1 = auto-detect, 0 = LTR, 1 = RTL bool applyBidiVisual(const char* utf8, std::string& out, int paragraphLevel = -1); diff --git a/lib/MiniBidi/bidiclasses.t b/lib/MiniBidi/bidiclasses.t index f300d3ae..10a0ccf6 100644 --- a/lib/MiniBidi/bidiclasses.t +++ b/lib/MiniBidi/bidiclasses.t @@ -1,15 +1,19 @@ -/* bidiclasses.t — bidi class table for CrossPoint Hebrew/English epub. +/* bidiclasses.t — bidi class table for CrossPoint RTL (Hebrew/Arabic) epub. * * Coverage rationale: - * Hebrew + English is the primary target. However, CrossPoint renders - * Latin and Cyrillic scripts for many other languages, so these MUST be - * classified as L (not fall through to ON) to avoid regression when they - * appear adjacent to Hebrew runs. + * Hebrew and Arabic-script languages (Arabic, Farsi, Urdu, Sindhi, Pashto, + * Kurdish) are the RTL targets. CrossPoint also renders Latin and Cyrillic + * scripts for many other languages, so these MUST be classified as L (not + * fall through to ON) to avoid regression when they appear adjacent to + * RTL runs. * * Scripts NOT in this table fall through to ON — correct per UAX#9 for - * scripts CrossPoint's fonts don't support (CJK, Arabic, Devanagari, etc.) + * scripts CrossPoint's fonts don't support (CJK, Devanagari, etc.) * ON is the right class for "unknown" — it behaves neutrally. * + * Arabic ranges are sourced from Unicode UCD extracted/DerivedBidiClass.txt + * (values verified against Unicode 17.0.0). + * * Entries sorted ascending by first (binary search requirement). */ @@ -84,6 +88,36 @@ {0x05D0, 0x05EA, R}, /* alef … tav */ {0x05F0, 0x05F4, R}, /* alternative forms + geresh/gershayim */ +/* ── Arabic / Perso-Arabic (DerivedBidiClass.txt) ───────────────────── */ +/* Letters are AL (Arabic Letter), harakat/marks are NSM, Arabic-Indic + digits are AN, extended (Farsi/Urdu) digits are EN. + 0x0606-0x0607, 0x060E-0x060F, 0x06DE, 0x06E9 are ON — fall through. */ +{0x0600, 0x0605, AN}, /* Arabic number signs (Cf) */ +{0x0608, 0x0608, AL}, /* Arabic ray */ +{0x0609, 0x060A, ET}, /* per mille / per ten thousand */ +{0x060B, 0x060B, AL}, /* afghani sign */ +{0x060C, 0x060C, CS}, /* Arabic comma */ +{0x060D, 0x060D, AL}, /* Arabic date separator */ +{0x0610, 0x061A, NSM}, /* honorifics + small marks */ +{0x061B, 0x061F, AL}, /* semicolon, ALM, end-of-text, hamza mark, question mark */ +{0x0620, 0x064A, AL}, /* core Arabic letters + tatweel */ +{0x064B, 0x065F, NSM}, /* harakat (fathatan … wavy hamza below) */ +{0x0660, 0x0669, AN}, /* Arabic-Indic digits ٠-٩ */ +{0x066A, 0x066A, ET}, /* Arabic percent sign */ +{0x066B, 0x066C, AN}, /* decimal / thousands separators */ +{0x066D, 0x066F, AL}, /* five-pointed star, dotless beh/qaf */ +{0x0670, 0x0670, NSM}, /* superscript alef */ +{0x0671, 0x06D5, AL}, /* extended letters: Farsi, Urdu, Sindhi, Pashto, Kurdish */ +{0x06D6, 0x06DC, NSM}, /* Quranic annotation marks */ +{0x06DD, 0x06DD, AN}, /* end of ayah */ +{0x06DF, 0x06E4, NSM}, +{0x06E5, 0x06E6, AL}, /* small waw / small yeh */ +{0x06E7, 0x06E8, NSM}, +{0x06EA, 0x06ED, NSM}, +{0x06EE, 0x06EF, AL}, /* dal/reh with inverted V */ +{0x06F0, 0x06F9, EN}, /* extended Arabic-Indic digits ۰-۹ (Farsi/Urdu) — EN per UCD */ +{0x06FA, 0x06FF, AL}, + /* ── Latin Extended Additional (L) ─────────────────────────────────── */ /* Covers accented chars for Vietnamese, Welsh, Romanian, etc. Not currently rendered by CrossPoint fonts, but costs only 2 table rows. */ @@ -110,6 +144,14 @@ {0x2069, 0x2069, PDI}, {0x206A, 0x206F, BN}, +/* ── Arabic presentation forms (output of do_shape()) ───────────────── */ +/* Contextual/ligature forms emitted by the shaper must classify as AL so + a reshaped line still resolves RTL. Ranges per DerivedBidiClass.txt; + 0xFBC3-0xFBD2 are ON — fall through. */ +{0xFB50, 0xFBC2, AL}, /* Presentation Forms-A: Perso-Arabic contextual forms */ +{0xFBD3, 0xFBFF, AL}, /* Presentation Forms-A: NG … Farsi Yeh forms */ +{0xFE70, 0xFE74, AL}, /* Presentation Forms-B: harakat isolated forms */ +{0xFE76, 0xFEFC, AL}, /* Presentation Forms-B: contextual forms + Lam-Alef ligatures */ /* ── Byte Order Mark ────────────────────────────────────────────────── */ {0xFEFF, 0xFEFF, BN}, diff --git a/lib/MiniBidi/minibidi.c b/lib/MiniBidi/minibidi.c index 26283702..68d90c26 100644 --- a/lib/MiniBidi/minibidi.c +++ b/lib/MiniBidi/minibidi.c @@ -127,6 +127,345 @@ ucschar mirror(ucschar c) { return p ? p->to : c; } +/* ═══════════════════════════════════════════════════════════════════════ + * Arabic contextual shaping — do_shape() + * + * Ported from mintty src/minibidi.c (https://github.com/mintty/mintty), + * original author Ahmad Khalifa (www.arabeyes.org), maintained by + * Thomas Wolff. MIT licence. + * + * CrossPoint deviations from the upstream code, each marked inline: + * 1. Joining-context lookups skip NSM marks (harakat). mintty stores + * combining marks out-of-line in terminal cells, so they never appear + * in its bidi_char array; in CrossPoint they are real array entries + * and must be transparent for joining (Unicode joining type T). + * 2. The Alef absorbed into a Lam-Alef ligature is overwritten with + * LIGATURE_PLACEHOLDER instead of a space: a terminal must keep the + * cell, a proportional-text renderer must drop the character. + * 3. The STYPE/SISOLATED macros became functions backed by a second + * lookup table covering Perso-Arabic letters outside mintty's native + * U+0621–U+064A range (Farsi پ چ ژ گ, Urdu ٹ ڈ ڑ ں ہ ے, plus Sindhi/ + * Pashto/Kurdish letters). Joining types are sourced from Unicode + * ArabicShaping.txt and presentation forms from UnicodeData.txt + * (Arabic Presentation Forms-A), both Unicode 17.0.0. Letters with a + * joining type but no presentation-form codepoints keep their base + * codepoint (neighbours still shape correctly around them). + * U+200C/U+200D also get their ArabicShaping.txt types (U and C) so + * that in-stream ZWJ/ZWNJ — which mintty never has in its array — + * affect adjacency the same way the joiners flags do. + * ═══════════════════════════════════════════════════════════════════════ */ + +/* Shaping Types (mintty) */ +enum { + SL, /* Left-Joining, doesn't exist in U+0600 - U+06FF */ + SR, /* Right-Joining, i.e. has Isolated, Final */ + SD, /* Dual-Joining, i.e. has Isolated, Final, Initial, Medial */ + SU, /* Non-Joining */ + SC /* Join-Causing, like U+0640 (TATWEEL) */ +}; + +typedef struct { + uchar type; + uchar form_b; /* isolated form = 0xFE00 + form_b (Presentation Forms-B) */ +} shape_node; + +/* Kept near the actual table, for verification. (mintty) */ +enum { SHAPE_FIRST = 0x621, SHAPE_LAST = 0x64A }; + +/* mintty's shapetypes[] — verbatim */ +static const shape_node shapetypes[] = { + /* index, Typ, Iso, Ligature Index */ + /* 621 */ {SU, 0x80}, + /* 622 */ {SR, 0x81}, + /* 623 */ {SR, 0x83}, + /* 624 */ {SR, 0x85}, + /* 625 */ {SR, 0x87}, + /* 626 */ {SD, 0x89}, + /* 627 */ {SR, 0x8D}, + /* 628 */ {SD, 0x8F}, + /* 629 */ {SR, 0x93}, + /* 62A */ {SD, 0x95}, + /* 62B */ {SD, 0x99}, + /* 62C */ {SD, 0x9D}, + /* 62D */ {SD, 0xA1}, + /* 62E */ {SD, 0xA5}, + /* 62F */ {SR, 0xA9}, + /* 630 */ {SR, 0xAB}, + /* 631 */ {SR, 0xAD}, + /* 632 */ {SR, 0xAF}, + /* 633 */ {SD, 0xB1}, + /* 634 */ {SD, 0xB5}, + /* 635 */ {SD, 0xB9}, + /* 636 */ {SD, 0xBD}, + /* 637 */ {SD, 0xC1}, + /* 638 */ {SD, 0xC5}, + /* 639 */ {SD, 0xC9}, + /* 63A */ {SD, 0xCD}, + /* 63B */ {SU, 0x0}, + /* 63C */ {SU, 0x0}, + /* 63D */ {SU, 0x0}, + /* 63E */ {SU, 0x0}, + /* 63F */ {SU, 0x0}, + /* 640 */ {SC, 0x0}, + /* 641 */ {SD, 0xD1}, + /* 642 */ {SD, 0xD5}, + /* 643 */ {SD, 0xD9}, + /* 644 */ {SD, 0xDD}, + /* 645 */ {SD, 0xE1}, + /* 646 */ {SD, 0xE5}, + /* 647 */ {SD, 0xE9}, + /* 648 */ {SR, 0xED}, + /* 649 */ {SR, 0xEF}, + /* 64A */ {SD, 0xF1}}; + +/* ── CrossPoint extension: joining types outside U+0621–U+064A ────────── + * Source: Unicode ArabicShaping.txt (joining type column). Ranges of + * letters sharing one type are collapsed. Sorted ascending (binary + * search). Anything not listed here or in shapetypes[] is SU. */ +static const struct { + ucschar first, last; + uchar type; +} xjointypes[] = { + {0x0620, 0x0620, SD}, /* kashmiri yeh */ + {0x066E, 0x066F, SD}, /* dotless beh/qaf */ + {0x0671, 0x0673, SR}, /* alef wasla + wavy-hamza alefs */ + {0x0675, 0x0677, SR}, /* high-hamza alef/waw */ + {0x0678, 0x0687, SD}, /* high-hamza yeh, beh group (ٹ ٺ ٻ … پ), hah group (چ ڇ) */ + {0x0688, 0x0699, SR}, /* dal group (ڈ ڊ ڌ …), reh group (ڑ ړ ژ …) */ + {0x069A, 0x06BF, SD}, /* seen/sad/feh/qaf/kaf/gaf/lam/noon groups (ک گ ں ھ …) */ + {0x06C0, 0x06C0, SR}, /* heh with yeh above */ + {0x06C1, 0x06C2, SD}, /* heh goal (ہ) */ + {0x06C3, 0x06CB, SR}, /* teh marbuta goal, waw group (ۆ ۇ ۋ …) */ + {0x06CC, 0x06CC, SD}, /* farsi yeh (ی) */ + {0x06CD, 0x06CD, SR}, /* yeh with tail */ + {0x06CE, 0x06CE, SD}, /* farsi yeh with V (Kurdish ێ) */ + {0x06CF, 0x06CF, SR}, /* waw with dot above */ + {0x06D0, 0x06D1, SD}, /* e (Pashto ې), yeh three dots below */ + {0x06D2, 0x06D3, SR}, /* yeh barree (ے ۓ) */ + {0x06D5, 0x06D5, SR}, /* ae (Kurdish ە) */ + {0x06EE, 0x06EF, SR}, /* dal/reh with inverted V */ + {0x06FA, 0x06FC, SD}, /* sheen/dad/ghain with dot below */ + {0x06FF, 0x06FF, SD}, /* heh with inverted V */ + {0x200C, 0x200C, SU}, /* ZWNJ — joining type U per ArabicShaping.txt */ + {0x200D, 0x200D, SC}, /* ZWJ — joining type C per ArabicShaping.txt */ +}; + +/* ── CrossPoint extension: presentation forms outside U+0621–U+064A ───── + * Source: UnicodeData.txt, Arabic Presentation Forms-A (U+FB50–U+FBFF). + * forms = number of consecutive presentation forms allocated for the + * letter, always in the order isolated, final, initial, medial (matching + * the SHAPE_* offsets below): 2 = isolated+final, 4 = all. + * Letters absent here have no presentation forms and keep their base + * codepoint. Sorted ascending (binary search). */ +static const struct { + ucschar cp; + ucschar isolated; + uchar forms; +} xshapeforms[] = { + {0x0671, 0xFB50, 2}, /* alef wasla */ + {0x0679, 0xFB66, 4}, /* tteh (Urdu ٹ) */ + {0x067A, 0xFB5E, 4}, /* tteheh */ + {0x067B, 0xFB52, 4}, /* beeh (Sindhi ٻ) */ + {0x067E, 0xFB56, 4}, /* peh (Farsi پ) */ + {0x067F, 0xFB62, 4}, /* teheh */ + {0x0680, 0xFB5A, 4}, /* beheh (Sindhi ڀ) */ + {0x0683, 0xFB76, 4}, /* nyeh (Sindhi ڃ) */ + {0x0684, 0xFB72, 4}, /* dyeh (Sindhi ڄ) */ + {0x0686, 0xFB7A, 4}, /* tcheh (Farsi چ) */ + {0x0687, 0xFB7E, 4}, /* tcheheh (Sindhi ڇ) */ + {0x0688, 0xFB88, 2}, /* ddal (Urdu ڈ) */ + {0x068C, 0xFB84, 2}, /* dahal (Sindhi ڌ) */ + {0x068D, 0xFB82, 2}, /* ddahal (Sindhi ڍ) */ + {0x068E, 0xFB86, 2}, /* dul (Sindhi ڎ) */ + {0x0691, 0xFB8C, 2}, /* rreh (Urdu ڑ) */ + {0x0698, 0xFB8A, 2}, /* jeh (Farsi ژ) */ + {0x06A4, 0xFB6A, 4}, /* veh (Kurdish ڤ) */ + {0x06A6, 0xFB6E, 4}, /* peheh (Sindhi ڦ) */ + {0x06A9, 0xFB8E, 4}, /* keheh (Farsi/Urdu ک) */ + {0x06AD, 0xFBD3, 4}, /* ng */ + {0x06AF, 0xFB92, 4}, /* gaf (Farsi/Urdu گ) */ + {0x06B1, 0xFB9A, 4}, /* ngoeh (Sindhi ڱ) */ + {0x06B3, 0xFB96, 4}, /* gueh (Sindhi ڳ) */ + {0x06BA, 0xFB9E, 2}, /* noon ghunna (Urdu ں) — dual-joining but only + isolated+final forms exist; initial/medial + contexts keep the base codepoint */ + {0x06BB, 0xFBA0, 4}, /* rnoon (Sindhi ڻ) */ + {0x06BE, 0xFBAA, 4}, /* heh doachashmee (Urdu ھ) */ + {0x06C0, 0xFBA4, 2}, /* heh with yeh above */ + {0x06C1, 0xFBA6, 4}, /* heh goal (Urdu ہ) */ + {0x06C5, 0xFBE0, 2}, /* kirghiz oe */ + {0x06C6, 0xFBD9, 2}, /* oe (Kurdish ۆ) */ + {0x06C7, 0xFBD7, 2}, /* u (ۇ) */ + {0x06C8, 0xFBDB, 2}, /* yu */ + {0x06C9, 0xFBE2, 2}, /* kirghiz yu */ + {0x06CB, 0xFBDE, 2}, /* ve */ + {0x06CC, 0xFBFC, 4}, /* farsi yeh (Farsi/Urdu ی) */ + {0x06D0, 0xFBE4, 4}, /* e (Pashto ې) */ + {0x06D2, 0xFBAE, 2}, /* yeh barree (Urdu ے) */ + {0x06D3, 0xFBB0, 2}, /* yeh barree with hamza above (ۓ) */ +}; + +/* Contextual form offsets from the isolated form — identical ordering in + Presentation Forms-A and -B, matching mintty's SFINAL/SINITIAL/SMEDIAL + (+1/+2/+3) macros. */ +enum { SHAPE_ISOLATED = 0, SHAPE_FINAL = 1, SHAPE_INITIAL = 2, SHAPE_MEDIAL = 3 }; + +/* STYPE equivalent (mintty macro → function to add the extended table) */ +static uchar stype(ucschar c) { + if (c >= SHAPE_FIRST && c <= SHAPE_LAST) return shapetypes[c - SHAPE_FIRST].type; + + int i = -1, j = lengthof(xjointypes); + while (j - i > 1) { + int k = (i + j) / 2; + if (c < xjointypes[k].first) + j = k; + else if (c > xjointypes[k].last) + i = k; + else + return xjointypes[k].type; + } + return SU; +} + +/* SISOLATED/SFINAL/SINITIAL/SMEDIAL equivalent. + form is one of the SHAPE_* offsets; returns c unchanged when the letter + has no presentation form allocated for that context. */ +static ucschar shape_form(ucschar c, uchar form) { + if (c >= SHAPE_FIRST && c <= SHAPE_LAST) { + const uchar form_b = shapetypes[c - SHAPE_FIRST].form_b; + return form_b ? 0xFE00 + form_b + form : c; + } + + int i = -1, j = lengthof(xshapeforms); + while (j - i > 1) { + int k = (i + j) / 2; + if (c < xshapeforms[k].cp) + j = k; + else if (c > xshapeforms[k].cp) + i = k; + else + return form < xshapeforms[k].forms ? xshapeforms[k].isolated + form : c; + } + return c; +} + +/* CrossPoint deviation 1: NSM marks (harakat) sit between letters in our + array but are transparent for joining (Unicode joining type T). These + helpers find the effective joining neighbour. */ +static int next_non_nsm(const bidi_char* line, int i, int count) { + int j = i + 1; + while (j < count && bidi_class(line[j].wc) == NSM) j++; + return j; +} + +static int prev_non_nsm(const bidi_char* line, int i) { + int j = i - 1; + while (j >= 0 && bidi_class(line[j].wc) == NSM) j--; + return j; +} + +/* The Main shaping function (mintty, structure preserved). + * + * line: visual-order buffer — must have been passed through do_bidi() first + * to: output buffer for the shaped data + * count: number of characters in line + */ +int do_shape(bidi_char* line, bidi_char* to, int count) { + for (int i = 0; i < count; i++) { + to[i] = line[i]; + int tempShape = stype(line[i].wc); + switch (tempShape) { + case SR: { /* Right-Joining, i.e. has Isolated, Final */ + const int nx = next_non_nsm(line, i, count); /* deviation 1: was i + 1 */ + tempShape = (nx < count) ? stype(line[nx].wc) : SU; + if (tempShape == SL || tempShape == SD || tempShape == SC) + to[i].wc = shape_form(line[i].wc, SHAPE_FINAL); + else + to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED); + break; + } + case SD: { /* Dual-Joining, i.e. has Isolated, Final, Initial, Medial */ + const int nx = next_non_nsm(line, i, count); /* deviation 1: was i + 1 */ + const int pv = prev_non_nsm(line, i); /* deviation 1: was i - 1 */ + + /* Make Ligatures */ + tempShape = (nx < count) ? stype(line[nx].wc) : SU; + if (line[i].wc == 0x644) { /* Lam: the Alef variant is at pv (visually left) */ + int ligFlag = 0; + switch (pv >= 0 ? line[pv].wc : 0) { + case 0x622: /* Alef with Madda above → لآ */ + ligFlag = 1; + to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEF6 : 0xFEF5; + break; + case 0x623: /* Alef with Hamza above → لأ */ + ligFlag = 1; + to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEF8 : 0xFEF7; + break; + case 0x625: /* Alef with Hamza below → لإ */ + ligFlag = 1; + to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEFA : 0xFEF9; + break; + case 0x627: /* Alef → لا */ + ligFlag = 1; + to[i].wc = (tempShape == SL || tempShape == SD || tempShape == SC) ? 0xFEFC : 0xFEFB; + break; + } + if (ligFlag) { + to[pv].wc = LIGATURE_PLACEHOLDER; /* deviation 2: mintty writes 0x20 */ + break; + } + } + + /* Arabic joining formatters: adapt forms (mintty) */ + const uchar joiners = line[i].joiners & 0xF; + const uchar prevjoiners = line[i].joiners >> 4; + if (prevjoiners == ZWNJ) { + /* backward join blocked; initial only if the visually-right + (logically next) neighbour joins, else isolated */ + tempShape = (pv >= 0) ? stype(line[pv].wc) : SU; + if (tempShape == SR || tempShape == SD || tempShape == SC) + to[i].wc = shape_form(line[i].wc, SHAPE_INITIAL); + else + to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED); + } else if (prevjoiners == (ZWJ | ZWNJ)) { + to[i].wc = shape_form(line[i].wc, SHAPE_MEDIAL); + } else if (prevjoiners == ZWJ) { + to[i].wc = shape_form(line[i].wc, SHAPE_FINAL); + } else if (joiners & ZWNJ) { + /* forward join blocked; final only if the visually-left + (logically previous) neighbour joins, else isolated — + tempShape still holds stype(nx) from above */ + if (tempShape == SL || tempShape == SD || tempShape == SC) + to[i].wc = shape_form(line[i].wc, SHAPE_FINAL); + else + to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED); + } else if (tempShape == SL || tempShape == SD || tempShape == SC) { + /* visually-right neighbour joins → final, or medial if the + visually-left neighbour joins too */ + tempShape = (pv >= 0) ? stype(line[pv].wc) : SU; + if (tempShape == SR || tempShape == SD || tempShape == SC) + to[i].wc = shape_form(line[i].wc, SHAPE_MEDIAL); + else + to[i].wc = shape_form(line[i].wc, SHAPE_FINAL); + } else { + /* visually-right neighbour doesn't join → isolated, or initial + if the visually-left neighbour joins */ + tempShape = (pv >= 0) ? stype(line[pv].wc) : SU; + if (tempShape == SR || tempShape == SD || tempShape == SC) + to[i].wc = shape_form(line[i].wc, SHAPE_INITIAL); + else + to[i].wc = shape_form(line[i].wc, SHAPE_ISOLATED); + } + break; + } + default: + /* SU, SL, SC and non-Arabic characters pass through unchanged */ + break; + } + } + return 1; +} + /* ═══════════════════════════════════════════════════════════════════════ * Directional Status Stack * (replaces GCC nested functions — ESP32C3 has no executable stack) diff --git a/lib/MiniBidi/minibidi.h b/lib/MiniBidi/minibidi.h index 26555d1b..7f2aa4a0 100644 --- a/lib/MiniBidi/minibidi.h +++ b/lib/MiniBidi/minibidi.h @@ -5,8 +5,10 @@ * minibidi.h — standalone header for ESP32C3 BiDi calculations * * Derived from [mintty](https://github.com/mintty/mintty/) (Thomas Wolff, MIT licence). - * Stripped of: Arabic shaping, box-drawing mirror, terminal dependencies, - * GCC nested functions, VLAs, and non-Hebrew/English Unicode data. + * Includes: UAX#9 bidi (do_bidi) and Arabic contextual shaping (do_shape), + * both ported from mintty src/minibidi.c (Ahmad Khalifa, Thomas Wolff). + * Stripped of: box-drawing mirror, terminal dependencies, GCC nested + * functions, VLAs, and Unicode data for scripts CrossPoint doesn't render. */ #include @@ -34,13 +36,18 @@ typedef uint32_t ucschar; /* Unicode codepoint; BMP-only content fits uint16_t #define BIDI_MAX_LINE 128 /* ── bidi_char ───────────────────────────────────────────────────────── */ -/* origwc: the codepoint as it came from the epub text stream - wc: working codepoint (may be replaced by mirrored form after do_bidi) - index: original logical position, so the caller can reorder glyphs */ +/* origwc: the codepoint as it came from the epub text stream + wc: working codepoint (may be replaced by mirrored form after + do_bidi, or by an Arabic contextual form after do_shape) + index: original logical position, so the caller can reorder glyphs + joiners: ZWJ/ZWNJ context for Arabic shaping, mintty layout: + low nibble = joiners that logically FOLLOW this character, + high nibble = joiners that logically PRECEDE this character */ typedef struct { ucschar origwc; ucschar wc; uint16_t index; + uint8_t joiners; } bidi_char; /* ── Bidi character classes (UAX #9) ────────────────────────────────── */ @@ -71,6 +78,20 @@ enum { PDI, /* Pop Directional Isolate */ }; +/* ── Arabic joining formatter flags (bidi_char.joiners nibbles) ─────── */ +/* Values match mintty's minibidi.h: ZWNJ 0x01, ZWJ 0x02. do_shape() + compares nibble values directly, so these must not be changed. */ +enum { + ZWNJ = 0x01, /* U+200C ZERO WIDTH NON-JOINER */ + ZWJ = 0x02, /* U+200D ZERO WIDTH JOINER */ +}; + +/* Sentinel written by do_shape() over the Alef absorbed into a Lam-Alef + ligature. Callers must filter it out when emitting shaped text. + (Upstream mintty writes a space instead — a terminal must keep the cell; + a proportional-text renderer must drop the character entirely.) */ +#define LIGATURE_PLACEHOLDER 0xFFFFu + /* ── Public API ──────────────────────────────────────────────────────── */ /* @@ -94,6 +115,32 @@ bool is_rtl_class(uchar bc); */ ucschar mirror(ucschar ch); +/* + * do_shape(line, to, count) + * + * Applies Arabic contextual shaping (and Lam-Alef ligation) to + * `line[0..count-1]`, writing the result to `to[0..count-1]`. + * + * MUST be called AFTER do_bidi(): the algorithm resolves joining from + * VISUAL adjacency (line[i-1] is the visually-left neighbour, line[i+1] + * the visually-right one), exactly like upstream mintty. + * + * line: visual-order input; the joiners field must be populated by the + * caller (in logical order, before do_bidi) for ZWJ/ZWNJ support + * to: output buffer, same size as line; non-Arabic entries are copied + * through unchanged. An Alef absorbed by a Lam-Alef ligature is + * replaced with LIGATURE_PLACEHOLDER — filter it on emission. + * count: number of characters (≤ BIDI_MAX_LINE) + * + * Returns 1. + * + * Ported from mintty src/minibidi.c (Ahmad Khalifa, Thomas Wolff, + * MIT licence), https://github.com/mintty/mintty — with CrossPoint + * extensions for Perso-Arabic letters and in-stream diacritics, see + * minibidi.c for details. + */ +int do_shape(bidi_char* line, bidi_char* to, int count); + /* * do_bidi(autodir, paragraphLevel, line, count) * diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 17c46643..db3a3f35 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.16) -project(crosspoint_reader_tests CXX) +project(crosspoint_reader_tests C CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -43,3 +43,5 @@ add_subdirectory(release_json_parser) add_subdirectory(differential_rounding) add_subdirectory(hyphenation_eval) add_subdirectory(utf8_compose) +add_subdirectory(minibidi_arabic) +add_subdirectory(combining_marks) diff --git a/test/combining_marks/CMakeLists.txt b/test/combining_marks/CMakeLists.txt new file mode 100644 index 00000000..c9a95225 --- /dev/null +++ b/test/combining_marks/CMakeLists.txt @@ -0,0 +1,14 @@ +add_executable(CombiningMarkAnchorTest + CombiningMarkAnchorTest.cpp +) + +target_include_directories(CombiningMarkAnchorTest PRIVATE + ${REPO_ROOT}/lib/EpdFont +) + +target_link_libraries(CombiningMarkAnchorTest PRIVATE + crosspoint_test_common + GTest::gtest_main +) + +gtest_discover_tests(CombiningMarkAnchorTest) diff --git a/test/combining_marks/CombiningMarkAnchorTest.cpp b/test/combining_marks/CombiningMarkAnchorTest.cpp new file mode 100644 index 00000000..ce7a4360 --- /dev/null +++ b/test/combining_marks/CombiningMarkAnchorTest.cpp @@ -0,0 +1,73 @@ +#include + +#include "lib/EpdFont/EpdFontData.h" + +// ============================================================================ +// Anchor selection and placement for combining marks without GPOS tables. +// +// Hebrew niqqud whose identity depends on position must not use the default +// centre-and-raise heuristic: dagesh sits inside the letter body, the +// shin/sin dots sit over the letter's right/left arm, and holam hangs over +// the left corner (see PR #2541 review feedback). +// ============================================================================ + +using combiningMark::Anchor; +using combiningMark::anchorFor; +using combiningMark::anchorOver; +using combiningMark::anchorOverRotated90CW; +using combiningMark::raiseAboveBase; + +TEST(AnchorFor, PositionSensitiveNiqqud) { + EXPECT_EQ(anchorFor(0x05BC), Anchor::CenterNative); // dagesh/mapiq + EXPECT_EQ(anchorFor(0x05BA), Anchor::CenterNative); // holam haser for vav + EXPECT_EQ(anchorFor(0x05C1), Anchor::RightNative); // shin dot + EXPECT_EQ(anchorFor(0x05C2), Anchor::LeftNative); // sin dot + EXPECT_EQ(anchorFor(0x05B9), Anchor::LeftNative); // holam +} + +TEST(AnchorFor, EverythingElseKeepsCentreRaisedDefault) { + EXPECT_EQ(anchorFor(0x05B0), Anchor::CenterRaised); // Hebrew sheva + EXPECT_EQ(anchorFor(0x05B7), Anchor::CenterRaised); // Hebrew patach + EXPECT_EQ(anchorFor(0x064E), Anchor::CenterRaised); // Arabic fatha + EXPECT_EQ(anchorFor(0x0651), Anchor::CenterRaised); // Arabic shadda + EXPECT_EQ(anchorFor(0x0301), Anchor::CenterRaised); // combining acute +} + +// Base glyph: cursor 100, left 1, width 12. Mark: left 2, width 4. +TEST(AnchorOver, HorizontalPlacementPerAnchor) { + // Centered: base bitmap spans [101, 113), centre 107; mark starts at 105. + EXPECT_EQ(anchorOver(Anchor::CenterRaised, 100, 1, 12, 2, 4), 105 - 2); + EXPECT_EQ(anchorOver(Anchor::CenterNative, 100, 1, 12, 2, 4), 105 - 2); + // Left-aligned: mark bitmap starts at the base bitmap's left edge (101). + EXPECT_EQ(anchorOver(Anchor::LeftNative, 100, 1, 12, 2, 4), 101 - 2); + // Right-aligned: mark bitmap ends at the base bitmap's right edge (113). + EXPECT_EQ(anchorOver(Anchor::RightNative, 100, 1, 12, 2, 4), 109 - 2); +} + +// The rotated coordinate system inverts every left/width term, so the mark's +// offset from the base cursor must be the exact mirror of the unrotated one. +TEST(AnchorOverRotated90CW, MirrorsUnrotatedOffsets) { + for (const Anchor anchor : {Anchor::CenterRaised, Anchor::CenterNative, Anchor::LeftNative, Anchor::RightNative}) { + const int offset = anchorOver(anchor, 100, 1, 12, 2, 4) - 100; + EXPECT_EQ(anchorOverRotated90CW(anchor, 100, 1, 12, 2, 4), 100 - offset); + } +} + +TEST(RaiseAboveBase, NativeAnchorsKeepFontDesignedHeight) { + // A dagesh-like dot designed inside the letter body (top 8 of a base whose + // top is 12) must NOT be hoisted above the letter. + EXPECT_EQ(raiseAboveBase(Anchor::CenterNative, 8, 3, 12), 0); + // Shin/sin dots overlapping the letter's top must not be pushed clear. + EXPECT_EQ(raiseAboveBase(Anchor::RightNative, 13, 3, 12), 0); + EXPECT_EQ(raiseAboveBase(Anchor::LeftNative, 13, 3, 12), 0); +} + +TEST(RaiseAboveBase, CentreRaisedBehaviourUnchanged) { + // Above-baseline mark colliding with the base: raised to restore a 1px gap. + // gap = markTop - markHeight - baseTop = 10 - 3 - 12 = -5 -> raise 6. + EXPECT_EQ(raiseAboveBase(Anchor::CenterRaised, 10, 3, 12), 6); + // Already clear of the base: no raise. + EXPECT_EQ(raiseAboveBase(Anchor::CenterRaised, 16, 3, 12), 0); + // Below-baseline mark (kasra, cedilla): stays at font-native position. + EXPECT_EQ(raiseAboveBase(Anchor::CenterRaised, 2, 4, 12), 0); +} diff --git a/test/minibidi_arabic/CMakeLists.txt b/test/minibidi_arabic/CMakeLists.txt new file mode 100644 index 00000000..3208e7a5 --- /dev/null +++ b/test/minibidi_arabic/CMakeLists.txt @@ -0,0 +1,19 @@ +add_executable(MiniBidiArabicTest + MiniBidiArabicTest.cpp + ${REPO_ROOT}/lib/MiniBidi/minibidi.c + ${REPO_ROOT}/lib/MiniBidi/BidiUtils.cpp + ${REPO_ROOT}/lib/Utf8/Utf8.cpp +) + +target_include_directories(MiniBidiArabicTest PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/stubs # no-op Logging.h (real one needs Arduino) + ${REPO_ROOT}/lib/MiniBidi + ${REPO_ROOT}/lib/Utf8 +) + +target_link_libraries(MiniBidiArabicTest PRIVATE + crosspoint_test_common + GTest::gtest_main +) + +gtest_discover_tests(MiniBidiArabicTest) diff --git a/test/minibidi_arabic/MiniBidiArabicTest.cpp b/test/minibidi_arabic/MiniBidiArabicTest.cpp new file mode 100644 index 00000000..2215b29c --- /dev/null +++ b/test/minibidi_arabic/MiniBidiArabicTest.cpp @@ -0,0 +1,254 @@ +// MiniBidiArabicTest — Arabic/Farsi/Urdu bidi + contextual shaping. +// +// Exercises the full BidiUtils::applyBidiVisual() pipeline (the single code +// path shared by GfxRenderer::getTextWidth() and drawText()): UAX#9 +// reordering (do_bidi) followed by mintty-ported Arabic shaping (do_shape). +// +// Inputs are built from codepoint arrays in LOGICAL order; expectations are +// codepoint arrays in VISUAL order (left to right, as drawn on screen). +// Presentation-form expectations were derived by hand from Unicode +// ArabicShaping.txt joining types and the Arabic Presentation Forms-A/B +// blocks of UnicodeData.txt. + +#include + +#include +#include +#include + +#include "BidiUtils.h" +#include "Utf8.h" + +namespace { + +std::string encode(const std::vector& codepoints) { + std::string utf8; + for (const uint32_t cp : codepoints) utf8AppendCodepoint(cp, utf8); + return utf8; +} + +std::vector decode(const std::string& utf8) { + std::vector codepoints; + auto* p = reinterpret_cast(utf8.c_str()); + while (*p) { + const uint32_t cp = utf8NextCodepoint(&p); + if (!cp) break; + codepoints.push_back(cp); + } + return codepoints; +} + +// Runs the shared bidi+shape pipeline with auto-detected paragraph direction +// and returns the visual-order codepoints. +std::vector shapeVisual(const std::vector& logical) { + const std::string in = encode(logical); + std::string out; + if (!BidiUtils::applyBidiVisual(in.c_str(), out, /*paragraphLevel=*/-1)) { + return decode(in); // pipeline declined: unchanged text + } + return decode(out); +} + +using CP = std::vector; + +} // namespace + +/* ── Core Arabic contextual forms ────────────────────────────────────── */ + +// A single letter renders in its isolated presentation form. +TEST(ArabicShaping, SingleLetterIsolated) { + EXPECT_EQ(shapeVisual({0x0628}), (CP{0xFE8F})); // ب +} + +// دار: dal and alef are right-joining, so no letter connects forward — +// all three stay isolated. Visual order is reversed. +TEST(ArabicShaping, RightJoinersStayIsolated) { + EXPECT_EQ(shapeVisual({0x062F, 0x0627, 0x0631}), (CP{0xFEAD, 0xFE8D, 0xFEA9})); +} + +// بيت: initial + medial + final chain of dual-joining letters. +TEST(ArabicShaping, DualJoiningChain) { + EXPECT_EQ(shapeVisual({0x0628, 0x064A, 0x062A}), (CP{0xFE96, 0xFEF4, 0xFE91})); +} + +// محمد: medial forms across four letters ending in right-joining dal. +TEST(ArabicShaping, MedialForms) { + EXPECT_EQ(shapeVisual({0x0645, 0x062D, 0x0645, 0x062F}), (CP{0xFEAA, 0xFEE4, 0xFEA4, 0xFEE3})); +} + +// Tatweel (U+0640) is join-causing and passes through unchanged. +TEST(ArabicShaping, TatweelJoinsBothSides) { + EXPECT_EQ(shapeVisual({0x0628, 0x0640, 0x062A}), (CP{0xFE96, 0x0640, 0xFE91})); +} + +/* ── Lam-Alef ligatures ──────────────────────────────────────────────── */ + +// All four Lam-Alef variants, isolated (nothing joins into the Lam). +// The absorbed Alef must be collapsed out of the output entirely. +TEST(ArabicShaping, LamAlefIsolatedVariants) { + EXPECT_EQ(shapeVisual({0x0644, 0x0627}), (CP{0xFEFB})); // لا + EXPECT_EQ(shapeVisual({0x0644, 0x0622}), (CP{0xFEF5})); // لآ + EXPECT_EQ(shapeVisual({0x0644, 0x0623}), (CP{0xFEF7})); // لأ + EXPECT_EQ(shapeVisual({0x0644, 0x0625}), (CP{0xFEF9})); // لإ +} + +// All four Lam-Alef variants in final form (a joining letter precedes Lam). +TEST(ArabicShaping, LamAlefFinalVariants) { + EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0627}), (CP{0xFEFC, 0xFE91})); + EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0622}), (CP{0xFEF6, 0xFE91})); + EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0623}), (CP{0xFEF8, 0xFE91})); + EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0625}), (CP{0xFEFA, 0xFE91})); +} + +// The specific bug flagged in PR #2398 review: a diacritic between Lam and +// Alef must not break the ligature. The fatha stays in the stream (a +// zero-advance overlay at render time); the Alef is absorbed. Per UAX#9 L3 +// the mark is emitted after the ligature it decorates. +TEST(ArabicShaping, LamAlefWithDiacriticBetween) { + EXPECT_EQ(shapeVisual({0x0644, 0x064E, 0x0627}), (CP{0xFEFB, 0x064E})); +} + +/* ── Harakat (diacritics) ────────────────────────────────────────────── */ + +// كَتَبَ fully vocalized: harakat are transparent for joining — the letter +// skeleton shapes exactly as كتب — and remain in the output stream. UAX#9 +// rule L3: each mark is emitted after its base, so the renderer can overlay +// it on the most recently drawn glyph. +TEST(ArabicShaping, VocalizedTextJoinsAcrossHarakat) { + EXPECT_EQ(shapeVisual({0x0643, 0x064E, 0x062A, 0x064E, 0x0628, 0x064E}), + (CP{0xFE90, 0x064E, 0xFE98, 0x064E, 0xFEDB, 0x064E})); +} + +// Stacked marks (shadda + fatha on one base) keep their logical order after +// the base, so above-base stacking renders bottom-up as authored. +TEST(ArabicShaping, StackedMarksFollowBaseInLogicalOrder) { + EXPECT_EQ(shapeVisual({0x0628, 0x0651, 0x064E}), (CP{0xFE8F, 0x0651, 0x064E})); +} + +// Hebrew niqqud rides the same L3 path: mark follows its base after the +// RTL run is reversed. +TEST(HebrewShaping, NiqqudFollowsBase) { + EXPECT_EQ(shapeVisual({0x05E9, 0x05B0, 0x05DC}), (CP{0x05DC, 0x05E9, 0x05B0})); +} + +/* ── Farsi extra letters ─────────────────────────────────────────────── */ + +// Peh joins like beh, using Presentation Forms-A codepoints. +TEST(FarsiShaping, PehContextualForms) { EXPECT_EQ(shapeVisual({0x067E, 0x067E}), (CP{0xFB57, 0xFB58})); } + +// گاز: gaf initial, alef final, jeh-group zain isolated. +TEST(FarsiShaping, GafWord) { EXPECT_EQ(shapeVisual({0x06AF, 0x0627, 0x0632}), (CP{0xFEAF, 0xFE8E, 0xFB94})); } + +// سیب: Farsi yeh (U+06CC) takes its medial Presentation Forms-A form. +TEST(FarsiShaping, FarsiYehMedial) { EXPECT_EQ(shapeVisual({0x0633, 0x06CC, 0x0628}), (CP{0xFE90, 0xFBFF, 0xFEB3})); } + +/* ── Urdu extra letters ──────────────────────────────────────────────── */ + +// ٹ (U+0679) and ڑ (U+0691) were specifically flagged in PR #2398 review as +// missing from a literal mintty port; ڈ (U+0688) completes the retroflex set. +TEST(UrduShaping, RetroflexLetters) { EXPECT_EQ(shapeVisual({0x0679, 0x0688, 0x0691}), (CP{0xFB8C, 0xFB89, 0xFB68})); } + +// میں: noon ghunna (U+06BA) is dual-joining but only has isolated/final +// presentation forms — final position works. +TEST(UrduShaping, NoonGhunnaFinal) { EXPECT_EQ(shapeVisual({0x0645, 0x06CC, 0x06BA}), (CP{0xFB9F, 0xFBFF, 0xFEE3})); } + +// ہے: heh goal initial + yeh barree final. +TEST(UrduShaping, HehGoalYehBarree) { EXPECT_EQ(shapeVisual({0x06C1, 0x06D2}), (CP{0xFBAF, 0xFBA8})); } + +/* ── Sindhi / Pashto / Kurdish samples ───────────────────────────────── */ + +// Sindhi ٻار: beeh (U+067B) initial form from Presentation Forms-A. +TEST(SindhiShaping, BeehInitial) { EXPECT_EQ(shapeVisual({0x067B, 0x0627, 0x0631}), (CP{0xFEAD, 0xFE8E, 0xFB54})); } + +// Pashto ښه: seen-with-dots (U+069A) has a joining type but NO presentation +// forms — it keeps its base codepoint while its neighbour still takes the +// correct joined form. +TEST(PashtoShaping, LetterWithoutPresentationFormsFallsBack) { + EXPECT_EQ(shapeVisual({0x069A, 0x0647}), (CP{0xFEEA, 0x069A})); +} + +// Pashto کور: keheh initial, waw final, reh isolated. +TEST(PashtoShaping, KehehWord) { EXPECT_EQ(shapeVisual({0x06A9, 0x0648, 0x0631}), (CP{0xFEAD, 0xFEEE, 0xFB90})); } + +// Kurdish ڕۆژ: all right-joining; ڕ (U+0695) has no presentation forms and +// stays as its base codepoint, ۆ and ژ take Presentation Forms-A isolated. +TEST(KurdishShaping, RightJoiningLetters) { + EXPECT_EQ(shapeVisual({0x0695, 0x06C6, 0x0698}), (CP{0xFB8A, 0xFBD9, 0x0695})); +} + +/* ── ZWJ / ZWNJ joining formatters ───────────────────────────────────── */ + +// ZWNJ blocks joining across it but leaves the outer sides to normal +// contextual rules (Farsi morphology, e.g. می‌خواهم). The formatter itself +// must be filtered from the output. Here meem joins into yeh (initial + +// final) while the ZWNJ keeps yeh from connecting to khah. +TEST(JoinerShaping, ZwnjBreaksJoin) { + // می‌خ: trailing khah has no forward partner → isolated. + EXPECT_EQ(shapeVisual({0x0645, 0x06CC, 0x200C, 0x062E}), (CP{0xFEA5, 0xFBFD, 0xFEE3})); + // می‌خو: waw follows khah → khah initial, waw final. + EXPECT_EQ(shapeVisual({0x0645, 0x06CC, 0x200C, 0x062E, 0x0648}), (CP{0xFEEE, 0xFEA7, 0xFBFD, 0xFEE3})); +} + +// ZWJ forces a join where none would occur: a lone beh followed by ZWJ takes +// initial form; preceded by ZWJ it takes final form. +TEST(JoinerShaping, ZwjForcesJoin) { + EXPECT_EQ(shapeVisual({0x0628, 0x200D}), (CP{0xFE91})); + EXPECT_EQ(shapeVisual({0x200D, 0x0628}), (CP{0xFE90})); +} + +/* ── Mixed-direction text ────────────────────────────────────────────── */ + +// كتاب abc 123 — RTL paragraph: the Arabic (shaped) ends up rightmost; +// "abc 123" resolves to a single LTR run (rule W7 turns the European +// numerals L after the strong L of "abc") and is placed as one block to +// its left, keeping internal left-to-right order. +TEST(MixedDirection, ArabicLatinNumerals) { + EXPECT_EQ(shapeVisual({0x0643, 0x062A, 0x0627, 0x0628, ' ', 'a', 'b', 'c', ' ', '1', '2', '3'}), + (CP{'a', 'b', 'c', ' ', '1', '2', '3', ' ', 0xFE8F, 0xFE8E, 0xFE98, 0xFEDB})); +} + +// كتاب ١٢٣ — Arabic-Indic digits keep logical order (leftmost run reads +// ١٢٣, not reversed). +TEST(MixedDirection, ArabicIndicNumerals) { + EXPECT_EQ(shapeVisual({0x0643, 0x062A, 0x0627, 0x0628, ' ', 0x0661, 0x0662, 0x0663}), + (CP{0x0661, 0x0662, 0x0663, ' ', 0xFE8F, 0xFE8E, 0xFE98, 0xFEDB})); +} + +// Watch-item regression probe for the "random spaces" report from #2398 +// testing: spaces must be neither duplicated nor dropped by the pipeline. +TEST(MixedDirection, SpaceCountPreserved) { + const CP visual = shapeVisual({0x0643, 0x062A, 0x0627, 0x0628, ' ', 0x0642, 0x0644, 0x0645, ' ', 'o', 'k'}); + int spaces = 0; + for (const uint32_t cp : visual) spaces += (cp == ' '); + EXPECT_EQ(spaces, 2); + EXPECT_EQ(visual.size(), 11u); // 9 letters + 2 spaces, nothing absorbed or invented +} + +/* ── Regression: existing behaviour unchanged ────────────────────────── */ + +// Hebrew reorders but never shapes. +TEST(Regression, HebrewUntouchedByShaper) { + EXPECT_EQ(shapeVisual({0x05E9, 0x05DC, 0x05D5, 0x05DD}), (CP{0x05DD, 0x05D5, 0x05DC, 0x05E9})); +} + +// Pure Latin text passes through unchanged. +TEST(Regression, LatinPassthrough) { EXPECT_EQ(shapeVisual({'h', 'e', 'l', 'l', 'o'}), (CP{'h', 'e', 'l', 'l', 'o'})); } + +/* ── isTransparentMark ───────────────────────────────────────────────── */ + +TEST(TransparentMark, RtlMarksAreTransparent) { + EXPECT_TRUE(BidiUtils::isTransparentMark(0x05B0)); // Hebrew sheva + EXPECT_TRUE(BidiUtils::isTransparentMark(0x0591)); // Hebrew accent etnahta + EXPECT_TRUE(BidiUtils::isTransparentMark(0x064E)); // Arabic fatha + EXPECT_TRUE(BidiUtils::isTransparentMark(0x0651)); // Arabic shadda + EXPECT_TRUE(BidiUtils::isTransparentMark(0x0670)); // superscript alef + EXPECT_TRUE(BidiUtils::isTransparentMark(0x06D6)); // Quranic annotation +} + +TEST(TransparentMark, LettersAndLatinMarksAreNot) { + EXPECT_FALSE(BidiUtils::isTransparentMark(0x0300)); // Latin combining grave — different path + EXPECT_FALSE(BidiUtils::isTransparentMark(0x0628)); // Arabic beh + EXPECT_FALSE(BidiUtils::isTransparentMark(0x05D0)); // Hebrew alef + EXPECT_FALSE(BidiUtils::isTransparentMark(0x0661)); // Arabic-Indic digit + EXPECT_FALSE(BidiUtils::isTransparentMark('a')); +} diff --git a/test/minibidi_arabic/stubs/Logging.h b/test/minibidi_arabic/stubs/Logging.h new file mode 100644 index 00000000..a69152df --- /dev/null +++ b/test/minibidi_arabic/stubs/Logging.h @@ -0,0 +1,8 @@ +#pragma once + +// Host-test stub for lib/Logging/Logging.h, which depends on Arduino's +// HardwareSerial and cannot compile on the host. Logging is a no-op here. + +#define LOG_ERR(origin, format, ...) +#define LOG_INF(origin, format, ...) +#define LOG_DBG(origin, format, ...)