Merge remote-tracking branch 'origin/develop' into feat-bluetooth

2026-07-14 04:20:50 -04:00
parent cecdbefa0e c5787d1cc1
commit 34e7ed249b
118 changed files with 22879 additions and 6746 deletions
@@ -44,16 +44,17 @@ void EpdFont::getTextBounds(const char* string, const int startX, const int star
      continue;
    }

-    const int raiseBy = isCombining ? combiningMark::raiseAboveBase(glyph->top, glyph->height, lastBaseTop) : 0;
+    const combiningMark::Anchor anchor = combiningMark::anchorFor(cp);
+    const int raiseBy = isCombining ? combiningMark::raiseAboveBase(anchor, glyph->top, glyph->height, lastBaseTop) : 0;

    if (!isCombining && prevCp != 0) {
      const auto kernFP = getKerning(prevCp, cp);  // 4.4 fixed-point kern
      lastBaseX += fp4::toPixel(prevAdvanceFP + kernFP);
    }

-    const int glyphBaseX =
-        isCombining ? combiningMark::centerOver(lastBaseX, lastBaseLeft, lastBaseWidth, glyph->left, glyph->width)
-                    : lastBaseX;
+    const int glyphBaseX = isCombining ? combiningMark::anchorOver(anchor, lastBaseX, lastBaseLeft, lastBaseWidth,
+                                                                   glyph->left, glyph->width)
+                                       : lastBaseX;
    const int glyphBaseY = startY - raiseBy;

    *minX = std::min(*minX, glyphBaseX + glyph->left);
@@ -36,24 +36,71 @@ namespace combiningMark {

 constexpr int MIN_GAP_PX = 1;

-/// Compute the cursor-X at which to render a combining mark so its bitmap
-/// is visually centered over the base glyph's bitmap.
-constexpr int centerOver(int baseCursorPos, int baseLeft, int baseWidth, int markLeft, int markWidth) {
-  return baseCursorPos + baseLeft + baseWidth / 2 - markWidth / 2 - markLeft;
+/// Placement of a mark relative to its base glyph.  The default heuristic —
+/// centered over the base, raised clear of its top — suits Latin diacritics
+/// and Arabic harakat, but misplaces the Hebrew niqqud whose identity depends
+/// on position: dagesh sits inside the letter body, the shin/sin dots
+/// distinguish the letter by sitting over its right/left arm, and holam hangs
+/// over the left corner.  "Native" anchors keep the glyph's font-designed
+/// height (which may overlap the base) instead of raising it.
+enum class Anchor : uint8_t {
+  CenterRaised,  ///< centered over the base, lifted above its top (default)
+  CenterNative,  ///< centered over the base at font-native height
+  RightNative,   ///< right edges aligned, font-native height
+  LeftNative,    ///< left edges aligned, font-native height
+};
+
+constexpr Anchor anchorFor(const uint32_t cp) {
+  switch (cp) {
+    case 0x05BC:  // dagesh / mapiq / shuruk dot: inside the letter body
+    case 0x05BA:  // holam haser for vav: straight above the vav stem
+      return Anchor::CenterNative;
+    case 0x05C1:  // shin dot: over the letter's right arm
+      return Anchor::RightNative;
+    case 0x05B9:  // holam: above the letter's left corner
+    case 0x05C2:  // sin dot: over the letter's left arm
+      return Anchor::LeftNative;
+    default:
+      return Anchor::CenterRaised;
+  }
 }

-/// Rotated-90CW variant of centerOver.  In the rotated coordinate system
+/// Horizontal offset from the base bitmap's left edge to the mark bitmap's
+/// left edge for a given anchor.
+constexpr int anchorShift(const Anchor anchor, const int baseWidth, const int markWidth) {
+  switch (anchor) {
+    case Anchor::LeftNative:
+      return 0;
+    case Anchor::RightNative:
+      return baseWidth - markWidth;
+    default:
+      return baseWidth / 2 - markWidth / 2;
+  }
+}
+
+/// Compute the cursor-X at which to render a combining mark so its bitmap
+/// lands at its anchor position over the base glyph's bitmap.
+constexpr int anchorOver(const Anchor anchor, const int baseCursorPos, const int baseLeft, const int baseWidth,
+                         const int markLeft, const int markWidth) {
+  return baseCursorPos + baseLeft + anchorShift(anchor, baseWidth, markWidth) - markLeft;
+}
+
+/// Rotated-90CW variant of anchorOver.  In the rotated coordinate system
 /// renderCharImpl uses (cursorY - left) instead of (cursorX + left), so
 /// every left/width term inverts sign.
-constexpr int centerOverRotated90CW(int baseCursorPos, int baseLeft, int baseWidth, int markLeft, int markWidth) {
-  return baseCursorPos - baseLeft - baseWidth / 2 + markWidth / 2 + markLeft;
+constexpr int anchorOverRotated90CW(const Anchor anchor, const int baseCursorPos, const int baseLeft,
+                                    const int baseWidth, const int markLeft, const int markWidth) {
+  return baseCursorPos - baseLeft - anchorShift(anchor, baseWidth, markWidth) + markLeft;
 }

 /// For combining marks that sit entirely above the baseline, compute how many
 /// pixels to raise the mark so there is at least MIN_GAP_PX between its bottom
 /// edge and the top of the base glyph.  Returns 0 for marks that extend to or
-/// below the baseline (e.g. cedilla, dot-below, ogonek).
-constexpr int raiseAboveBase(int markTop, int markHeight, int baseTop) {
+/// below the baseline (e.g. cedilla, dot-below, ogonek) and for anchors that
+/// keep the font-native height (dagesh must stay inside the letter, the
+/// shin/sin dots touch its arms).
+constexpr int raiseAboveBase(const Anchor anchor, const int markTop, const int markHeight, const int baseTop) {
+  if (anchor != Anchor::CenterRaised) return 0;
  if (markTop - markHeight <= 0) return 0;
  const int gap = markTop - markHeight - baseTop;
  return (gap < MIN_GAP_PX) ? (MIN_GAP_PX - gap) : 0;
@@ -61,6 +108,20 @@ constexpr int raiseAboveBase(int markTop, int markHeight, int baseTop) {

 }  // namespace combiningMark

+/// GCC/Clang (the ESP32 firmware toolchain) pack structs with __attribute__((packed)).
+/// MSVC (host unit tests) has no equivalent attribute and instead needs a #pragma pack
+/// region achieving the same 1-byte alignment. These macros keep the on-disk font layout
+/// identical across both toolchains.
+#if defined(_MSC_VER)
+#define EPD_PACKED_BEGIN __pragma(pack(push, 1))
+#define EPD_PACKED_END __pragma(pack(pop))
+#define EPD_PACKED_ATTR
+#else
+#define EPD_PACKED_BEGIN
+#define EPD_PACKED_END
+#define EPD_PACKED_ATTR __attribute__((packed))
+#endif
+
 /// Fixed-point conventions used by EpdGlyph and EpdFontData:
 ///   advanceX:   12.4 unsigned fixed-point in uint16_t  (use fp4::toPixel)
 ///   kernMatrix:  4.4 signed fixed-point in int8_t      (use fp4::toPixel)
@@ -95,17 +156,21 @@ typedef struct {

 /// Maps a codepoint to a kerning class ID, sorted by codepoint for binary search.
 /// Class IDs are 1-based; codepoints not in the table have implicit class 0 (no kerning).
+EPD_PACKED_BEGIN
 typedef struct {
  uint16_t codepoint;  ///< Unicode codepoint
  uint8_t classId;     ///< 1-based kerning class ID
-} __attribute__((packed)) EpdKernClassEntry;
+} EPD_PACKED_ATTR EpdKernClassEntry;
+EPD_PACKED_END

 /// Ligature substitution for a specific glyph pair, sorted by `pair` for binary search.
 /// `pair` encodes (leftCodepoint << 16 | rightCodepoint) for single-key lookup.
+EPD_PACKED_BEGIN
 typedef struct {
  uint32_t pair;        ///< Packed codepoint pair (left << 16 | right)
  uint32_t ligatureCp;  ///< Codepoint of the replacement ligature glyph
-} __attribute__((packed)) EpdLigaturePair;
+} EPD_PACKED_ATTR EpdLigaturePair;
+EPD_PACKED_END

 /// Data stored for FONT AS A WHOLE
 typedef struct {
@@ -1222,7 +1222,8 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun
 }

 template <typename Iter>
-int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask) {
+int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask,
+                                       const char* extraText) {
  if (!loaded_) return -1;
  styleMask = resolveStyleMask(styleMask);
  if (styleMask == 0) return 0;
@@ -1242,6 +1243,9 @@ int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace,
  for (auto it = begin; it != end && !hitCap; ++it) {
    hitCap = collectUniqueCodepoints(asCStr(*it), codepoints, cpCount, MAX_UNIQUE_CODEPOINTS);
  }
+  if (extraText && !hitCap) {
+    hitCap = collectUniqueCodepoints(extraText, codepoints, cpCount, MAX_UNIQUE_CODEPOINTS);
+  }

  if (includeSpace && std::none_of(codepoints, codepoints + cpCount, [](uint32_t c) { return c == ' '; }))
    codepoints[cpCount++] = ' ';
@@ -1259,12 +1263,13 @@ int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace,
  return totalMissed;
 }

-int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask) {
-  return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask);
+int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask, const char* extraText) {
+  return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask, extraText);
 }

-int SdCardFont::buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask) {
-  return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask);
+int SdCardFont::buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask,
+                                  const char* extraText) {
+  return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask, extraText);
 }

 // --- Stats ---
@@ -47,9 +47,12 @@ class SdCardFont {
  // Build a compact advance-only table for layout measurement.
  // Extracts ALL unique codepoints from words (no MAX_PAGE_GLYPHS cap),
  // batch-reads advanceX from SD, stores in a sorted per-style table.
+  // extraText: optional additional codepoints to warm in the same SD pass
+  // (e.g. shaped Arabic presentation forms the measurement path will look up).
  // Returns number of codepoints not found in font coverage.
-  int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F);
-  int buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask = 0x0F);
+  int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F, const char* extraText = nullptr);
+  int buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask = 0x0F,
+                        const char* extraText = nullptr);

  // Look up advanceX for a codepoint from the advance table.
  // Returns the 12.4 fixed-point advance, or 0 if not found.
@@ -140,11 +143,13 @@ class SdCardFont {

    // Full intervals loaded from file (kept in RAM for codepoint lookup)
    EpdUnicodeInterval* fullIntervals = nullptr;
+    EPD_PACKED_BEGIN
    struct BmpInterval16 {
      uint16_t first;
      uint16_t last;
      uint16_t offset;
-    } __attribute__((packed));
+    } EPD_PACKED_ATTR;
+    EPD_PACKED_END
    static_assert(sizeof(BmpInterval16) == 6, "BmpInterval16 must remain compact");
    BmpInterval16* bmpIntervals = nullptr;
    bool intervalsAreBmp16 = false;
@@ -262,7 +267,8 @@ class SdCardFont {
  int32_t findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint) const;
  int fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask);
  template <typename Iter>
-  int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask);
+  int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask,
+                             const char* extraText = nullptr);
  int prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint32_t cpCount, bool metadataOnly);

  // Global helpers
@@ -6,6 +6,8 @@
 !NotoSerif/**
 !NotoSans/
 !NotoSans/**
+!NotoSansArabic/
+!NotoSansArabic/**
 !NotoSansHebrew/
 !NotoSansHebrew/**
 !OpenDyslexic/
@@ -0,0 +1,93 @@
+Copyright 2022 The Noto Project Authors (https://github.com/notofonts/arabic)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+https://openfontlicense.org
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.
@@ -31,19 +31,48 @@ done
 UI_FONT_SIZES=(10 12)
 UI_FONT_STYLES=("Regular" "Bold")

+# Arabic glyphs for UI text (menus, file browser titles). The built-in fonts
+# must cover the *output* of MiniBidi's do_shape() — contextual presentation
+# forms — not base letters, or shaped UI text silently drops glyphs.
+# Curated for firmware-size budget: core Arabic (Presentation Forms-B,
+# incl. the Lam-Alef ligature forms) plus the Farsi/Urdu extra letters'
+# Presentation Forms-A blocks, the few characters shaping leaves at their
+# base codepoint, Arabic punctuation, and both digit sets. No harakat and
+# no Sindhi/Pashto/Kurdish forms — book text gets those from SD-card fonts.
+ARABIC_INTERVALS=(
+  --additional-intervals 0x060C,0x060C  # Arabic comma
+  --additional-intervals 0x061B,0x061B  # Arabic semicolon
+  --additional-intervals 0x061F,0x061F  # Arabic question mark
+  --additional-intervals 0x0621,0x0621  # hamza (non-joining, never shaped)
+  --additional-intervals 0x0640,0x0640  # tatweel
+  --additional-intervals 0x0660,0x0669  # Arabic-Indic digits
+  --additional-intervals 0x06BA,0x06BA  # noon ghunna base (initial/medial keep base cp)
+  --additional-intervals 0x06D4,0x06D4  # Urdu full stop
+  --additional-intervals 0x06F0,0x06F9  # extended Arabic-Indic digits (Farsi/Urdu)
+  --additional-intervals 0xFB56,0xFB59  # peh (Farsi)
+  --additional-intervals 0xFB66,0xFB69  # tteh (Urdu)
+  --additional-intervals 0xFB7A,0xFB7D  # tcheh (Farsi)
+  --additional-intervals 0xFB88,0xFB95  # ddal, jeh, rreh (Urdu), keheh, gaf (Farsi/Urdu)
+  --additional-intervals 0xFB9E,0xFB9F  # noon ghunna isolated/final (Urdu)
+  --additional-intervals 0xFBA6,0xFBB1  # heh goal, heh doachashmee, yeh barree(+hamza) (Urdu)
+  --additional-intervals 0xFBFC,0xFBFF  # farsi yeh (Farsi/Urdu)
+  --additional-intervals 0xFE80,0xFEFC  # Presentation Forms-B: core Arabic + Lam-Alef
+)
+
 for size in ${UI_FONT_SIZES[@]}; do
  for style in ${UI_FONT_STYLES[@]}; do
    font_name="ubuntu_${size}_$(echo $style | tr '[:upper:]' '[:lower:]')"
    font_path="../builtinFonts/source/Ubuntu/Ubuntu-${style}.ttf"
    hebrew_path="../builtinFonts/source/NotoSansHebrew/NotoSansHebrew-${style}.ttf"
+    arabic_path="../builtinFonts/source/NotoSansArabic/NotoSansArabic-${style}.ttf"
    # Ubuntu lacks the Latin Extended Additional block (U+1EA0-U+1EF9) used for
    # Vietnamese tone marks. Append a Vietnamese-only Ubuntu cut so those glyphs
    # are filled from it while every glyph Ubuntu already has stays unchanged
    # (fontstack is ordered by descending priority).
    viet_path="../builtinFonts/source/Ubuntu/Ubuntu-Vietnamese-${style}.ttf"
    output_path="../builtinFonts/${font_name}.h"
-    python fontconvert.py $font_name $size $font_path $hebrew_path $viet_path \
-      --additional-intervals 0x05D0,0x05EA > $output_path
+    python fontconvert.py $font_name $size $font_path $hebrew_path $arabic_path $viet_path \
+      --additional-intervals 0x05D0,0x05EA "${ARABIC_INTERVALS[@]}" > $output_path
    echo "Generated $output_path"
  done
 done
@@ -51,7 +80,8 @@ done
 python fontconvert.py notosans_8_regular 8 \
  ../builtinFonts/source/NotoSans/NotoSans-Regular.ttf \
  ../builtinFonts/source/NotoSansHebrew/NotoSansHebrew-Regular.ttf \
-  --additional-intervals 0x05D0,0x05EA > ../builtinFonts/notosans_8_regular.h
+  ../builtinFonts/source/NotoSansArabic/NotoSansArabic-Regular.ttf \
+  --additional-intervals 0x05D0,0x05EA "${ARABIC_INTERVALS[@]}" > ../builtinFonts/notosans_8_regular.h

 echo ""
 echo "Running compression verification..."