feat: Arabic/Farsi/Urdu bidi reordering and contextual shaping — PR 1/3 (#2541)

Co-authored-by: Uri Tauber <uritaube@gmail.com>

lib/EpdFont/EpdFont.cpp

@@ -44,16 +44,17 @@ void EpdFont::getTextBounds(const char* string, const int startX, const int star
       continue;
     }
 
-    const int raiseBy = isCombining ? combiningMark::raiseAboveBase(glyph->top, glyph->height, lastBaseTop) : 0;
+    const combiningMark::Anchor anchor = combiningMark::anchorFor(cp);
+    const int raiseBy = isCombining ? combiningMark::raiseAboveBase(anchor, glyph->top, glyph->height, lastBaseTop) : 0;
 
     if (!isCombining && prevCp != 0) {
       const auto kernFP = getKerning(prevCp, cp);  // 4.4 fixed-point kern
       lastBaseX += fp4::toPixel(prevAdvanceFP + kernFP);
     }
 
-    const int glyphBaseX =
-        isCombining ? combiningMark::centerOver(lastBaseX, lastBaseLeft, lastBaseWidth, glyph->left, glyph->width)
-                    : lastBaseX;
+    const int glyphBaseX = isCombining ? combiningMark::anchorOver(anchor, lastBaseX, lastBaseLeft, lastBaseWidth,
+                                                                   glyph->left, glyph->width)
+                                       : lastBaseX;
     const int glyphBaseY = startY - raiseBy;
 
     *minX = std::min(*minX, glyphBaseX + glyph->left);

lib/EpdFont/EpdFontData.h

@@ -36,24 +36,71 @@ namespace combiningMark {
 
 constexpr int MIN_GAP_PX = 1;
 
-/// Compute the cursor-X at which to render a combining mark so its bitmap
-/// is visually centered over the base glyph's bitmap.
-constexpr int centerOver(int baseCursorPos, int baseLeft, int baseWidth, int markLeft, int markWidth) {
-  return baseCursorPos + baseLeft + baseWidth / 2 - markWidth / 2 - markLeft;
+/// Placement of a mark relative to its base glyph.  The default heuristic —
+/// centered over the base, raised clear of its top — suits Latin diacritics
+/// and Arabic harakat, but misplaces the Hebrew niqqud whose identity depends
+/// on position: dagesh sits inside the letter body, the shin/sin dots
+/// distinguish the letter by sitting over its right/left arm, and holam hangs
+/// over the left corner.  "Native" anchors keep the glyph's font-designed
+/// height (which may overlap the base) instead of raising it.
+enum class Anchor : uint8_t {
+  CenterRaised,  ///< centered over the base, lifted above its top (default)
+  CenterNative,  ///< centered over the base at font-native height
+  RightNative,   ///< right edges aligned, font-native height
+  LeftNative,    ///< left edges aligned, font-native height
+};
+
+constexpr Anchor anchorFor(const uint32_t cp) {
+  switch (cp) {
+    case 0x05BC:  // dagesh / mapiq / shuruk dot: inside the letter body
+    case 0x05BA:  // holam haser for vav: straight above the vav stem
+      return Anchor::CenterNative;
+    case 0x05C1:  // shin dot: over the letter's right arm
+      return Anchor::RightNative;
+    case 0x05B9:  // holam: above the letter's left corner
+    case 0x05C2:  // sin dot: over the letter's left arm
+      return Anchor::LeftNative;
+    default:
+      return Anchor::CenterRaised;
+  }
 }
 
-/// Rotated-90CW variant of centerOver.  In the rotated coordinate system
+/// Horizontal offset from the base bitmap's left edge to the mark bitmap's
+/// left edge for a given anchor.
+constexpr int anchorShift(const Anchor anchor, const int baseWidth, const int markWidth) {
+  switch (anchor) {
+    case Anchor::LeftNative:
+      return 0;
+    case Anchor::RightNative:
+      return baseWidth - markWidth;
+    default:
+      return baseWidth / 2 - markWidth / 2;
+  }
+}
+
+/// Compute the cursor-X at which to render a combining mark so its bitmap
+/// lands at its anchor position over the base glyph's bitmap.
+constexpr int anchorOver(const Anchor anchor, const int baseCursorPos, const int baseLeft, const int baseWidth,
+                         const int markLeft, const int markWidth) {
+  return baseCursorPos + baseLeft + anchorShift(anchor, baseWidth, markWidth) - markLeft;
+}
+
+/// Rotated-90CW variant of anchorOver.  In the rotated coordinate system
 /// renderCharImpl uses (cursorY - left) instead of (cursorX + left), so
 /// every left/width term inverts sign.
-constexpr int centerOverRotated90CW(int baseCursorPos, int baseLeft, int baseWidth, int markLeft, int markWidth) {
-  return baseCursorPos - baseLeft - baseWidth / 2 + markWidth / 2 + markLeft;
+constexpr int anchorOverRotated90CW(const Anchor anchor, const int baseCursorPos, const int baseLeft,
+                                    const int baseWidth, const int markLeft, const int markWidth) {
+  return baseCursorPos - baseLeft - anchorShift(anchor, baseWidth, markWidth) + markLeft;
 }
 
 /// For combining marks that sit entirely above the baseline, compute how many
 /// pixels to raise the mark so there is at least MIN_GAP_PX between its bottom
 /// edge and the top of the base glyph.  Returns 0 for marks that extend to or
-/// below the baseline (e.g. cedilla, dot-below, ogonek).
-constexpr int raiseAboveBase(int markTop, int markHeight, int baseTop) {
+/// below the baseline (e.g. cedilla, dot-below, ogonek) and for anchors that
+/// keep the font-native height (dagesh must stay inside the letter, the
+/// shin/sin dots touch its arms).
+constexpr int raiseAboveBase(const Anchor anchor, const int markTop, const int markHeight, const int baseTop) {
+  if (anchor != Anchor::CenterRaised) return 0;
   if (markTop - markHeight <= 0) return 0;
   const int gap = markTop - markHeight - baseTop;
   return (gap < MIN_GAP_PX) ? (MIN_GAP_PX - gap) : 0;
@@ -61,6 +108,20 @@ constexpr int raiseAboveBase(int markTop, int markHeight, int baseTop) {
 
 }  // namespace combiningMark
 
+/// GCC/Clang (the ESP32 firmware toolchain) pack structs with __attribute__((packed)).
+/// MSVC (host unit tests) has no equivalent attribute and instead needs a #pragma pack
+/// region achieving the same 1-byte alignment. These macros keep the on-disk font layout
+/// identical across both toolchains.
+#if defined(_MSC_VER)
+#define EPD_PACKED_BEGIN __pragma(pack(push, 1))
+#define EPD_PACKED_END __pragma(pack(pop))
+#define EPD_PACKED_ATTR
+#else
+#define EPD_PACKED_BEGIN
+#define EPD_PACKED_END
+#define EPD_PACKED_ATTR __attribute__((packed))
+#endif
+
 /// Fixed-point conventions used by EpdGlyph and EpdFontData:
 ///   advanceX:   12.4 unsigned fixed-point in uint16_t  (use fp4::toPixel)
 ///   kernMatrix:  4.4 signed fixed-point in int8_t      (use fp4::toPixel)
@@ -95,17 +156,21 @@ typedef struct {
 
 /// Maps a codepoint to a kerning class ID, sorted by codepoint for binary search.
 /// Class IDs are 1-based; codepoints not in the table have implicit class 0 (no kerning).
+EPD_PACKED_BEGIN
 typedef struct {
   uint16_t codepoint;  ///< Unicode codepoint
   uint8_t classId;     ///< 1-based kerning class ID
-} __attribute__((packed)) EpdKernClassEntry;
+} EPD_PACKED_ATTR EpdKernClassEntry;
+EPD_PACKED_END
 
 /// Ligature substitution for a specific glyph pair, sorted by `pair` for binary search.
 /// `pair` encodes (leftCodepoint << 16 | rightCodepoint) for single-key lookup.
+EPD_PACKED_BEGIN
 typedef struct {
   uint32_t pair;        ///< Packed codepoint pair (left << 16 | right)
   uint32_t ligatureCp;  ///< Codepoint of the replacement ligature glyph
-} __attribute__((packed)) EpdLigaturePair;
+} EPD_PACKED_ATTR EpdLigaturePair;
+EPD_PACKED_END
 
 /// Data stored for FONT AS A WHOLE
 typedef struct {

lib/EpdFont/SdCardFont.cpp

@@ -1195,7 +1195,8 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun
 }
 
 template <typename Iter>
-int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask) {
+int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask,
+                                       const char* extraText) {
   if (!loaded_) return -1;
   styleMask = resolveStyleMask(styleMask);
   if (styleMask == 0) return 0;
@@ -1215,6 +1216,9 @@ int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace,
   for (auto it = begin; it != end && !hitCap; ++it) {
     hitCap = collectUniqueCodepoints(asCStr(*it), codepoints, cpCount, MAX_UNIQUE_CODEPOINTS);
   }
+  if (extraText && !hitCap) {
+    hitCap = collectUniqueCodepoints(extraText, codepoints, cpCount, MAX_UNIQUE_CODEPOINTS);
+  }
 
   if (includeSpace && std::none_of(codepoints, codepoints + cpCount, [](uint32_t c) { return c == ' '; }))
     codepoints[cpCount++] = ' ';
@@ -1232,12 +1236,13 @@ int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace,
   return totalMissed;
 }
 
-int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask) {
-  return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask);
+int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask, const char* extraText) {
+  return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask, extraText);
 }
 
-int SdCardFont::buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask) {
-  return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask);
+int SdCardFont::buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask,
+                                  const char* extraText) {
+  return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask, extraText);
 }
 
 // --- Stats ---

lib/EpdFont/SdCardFont.h

@@ -47,9 +47,12 @@ class SdCardFont {
   // Build a compact advance-only table for layout measurement.
   // Extracts ALL unique codepoints from words (no MAX_PAGE_GLYPHS cap),
   // batch-reads advanceX from SD, stores in a sorted per-style table.
+  // extraText: optional additional codepoints to warm in the same SD pass
+  // (e.g. shaped Arabic presentation forms the measurement path will look up).
   // Returns number of codepoints not found in font coverage.
-  int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F);
-  int buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask = 0x0F);
+  int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F, const char* extraText = nullptr);
+  int buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask = 0x0F,
+                        const char* extraText = nullptr);
 
   // Look up advanceX for a codepoint from the advance table.
   // Returns the 12.4 fixed-point advance, or 0 if not found.
@@ -140,11 +143,13 @@ class SdCardFont {
 
     // Full intervals loaded from file (kept in RAM for codepoint lookup)
     EpdUnicodeInterval* fullIntervals = nullptr;
+    EPD_PACKED_BEGIN
     struct BmpInterval16 {
       uint16_t first;
       uint16_t last;
       uint16_t offset;
-    } __attribute__((packed));
+    } EPD_PACKED_ATTR;
+    EPD_PACKED_END
     static_assert(sizeof(BmpInterval16) == 6, "BmpInterval16 must remain compact");
     BmpInterval16* bmpIntervals = nullptr;
     bool intervalsAreBmp16 = false;
@@ -249,7 +254,8 @@ class SdCardFont {
   int32_t findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint) const;
   int fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask);
   template <typename Iter>
-  int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask);
+  int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask,
+                             const char* extraText = nullptr);
   int prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint32_t cpCount, bool metadataOnly);
 
   // Global helpers