feat: Support for Korean line breaks and glyph spacing (#2288)

Co-authored-by: Uri Tauber <uritaube@gmail.com>

lib/EpdFont/EpdFont.cpp

@@ -101,6 +101,9 @@ static uint8_t lookupKernClass(const EpdKernClassEntry* entries, const uint16_t
 }
 
 int8_t EpdFont::getKerning(const uint32_t leftCp, const uint32_t rightCp) const {
+  if (utf8IsCjkBreakable(leftCp) || utf8IsCjkBreakable(rightCp)) {
+    return 0;
+  }
   if (!data->kernMatrix) {
     return 0;
   }

lib/EpdFont/SdCardFont.cpp

@@ -115,6 +115,9 @@ void SdCardFont::freeStyleAll(PerStyle& s) {
   freeStyleMiniData(s);
   delete[] s.fullIntervals;
   s.fullIntervals = nullptr;
+  delete[] s.bmpIntervals;
+  s.bmpIntervals = nullptr;
+  s.intervalsAreBmp16 = false;
   freeStyleKernLigatureData(s);
   s.present = false;
 }
@@ -516,59 +519,94 @@ bool SdCardFont::load(const char* path) {
   styleCount_ = styleCount;
   contentHash_ = hash;
 
-  // Load full intervals into RAM for each present style
+  // Load full intervals into RAM for each present style. BMP-only fonts with
+  // fewer than 65536 glyphs use a compact 6-byte interval table instead of the
+  // on-disk 12-byte table; large sparse CJK subsets otherwise keep tens of KB
+  // of always-resident heap just for lookup metadata.
   for (uint8_t i = 0; i < MAX_STYLES; i++) {
     auto& s = styles_[i];
     if (!s.present) continue;
 
-    s.fullIntervals = new (std::nothrow) EpdUnicodeInterval[s.header.intervalCount];
-    if (!s.fullIntervals) {
-      LOG_ERR("SDCF", "Failed to allocate %u intervals for style %u", s.header.intervalCount, i);
-      freeAll();
-      return false;
-    }
-
     if (!file.seekSet(s.intervalsFileOffset)) {
       LOG_ERR("SDCF", "Failed to seek to intervals for style %u", i);
       freeAll();
       return false;
     }
-    size_t intervalsBytes = s.header.intervalCount * sizeof(EpdUnicodeInterval);
-    if (file.read(reinterpret_cast<uint8_t*>(s.fullIntervals), intervalsBytes) != static_cast<int>(intervalsBytes)) {
-      LOG_ERR("SDCF", "Failed to read intervals for style %u", i);
-      freeAll();
-      return false;
-    }
 
     // Validate interval contents before any later code (findGlobalGlyphIndex,
     // glyph reads) trusts them. A malformed file could otherwise drive
     // out-of-range glyph indices into bogus on-disk reads.
-    {
-      uint32_t expectedOffset = 0;
-      uint32_t prevLast = 0;
+    bool canUseBmp16 = s.header.glyphCount <= UINT16_MAX;
+    uint32_t expectedOffset = 0;
+    uint32_t prevLast = 0;
+    EpdUnicodeInterval iv{};
+    for (uint32_t j = 0; j < s.header.intervalCount; ++j) {
+      if (file.read(reinterpret_cast<uint8_t*>(&iv), sizeof(iv)) != sizeof(iv)) {
+        LOG_ERR("SDCF", "Failed to read interval %u for style %u", j, i);
+        freeAll();
+        return false;
+      }
+      if (iv.first > iv.last) {
+        LOG_ERR("SDCF", "Style %u: invalid interval %u (first 0x%lX > last 0x%lX)", i, j,
+                static_cast<unsigned long>(iv.first), static_cast<unsigned long>(iv.last));
+        file.close();
+        freeAll();
+        return false;
+      }
+      const uint32_t span = iv.last - iv.first + 1;
+      const bool overlapsPrev = (j > 0 && iv.first <= prevLast);
+      const bool spanTooBig = (span > s.header.glyphCount);
+      const bool offsetMismatch = (iv.offset != expectedOffset);
+      const bool offsetOverruns = (iv.offset > s.header.glyphCount - span);
+      if (overlapsPrev || spanTooBig || offsetMismatch || offsetOverruns) {
+        LOG_ERR("SDCF", "Style %u: invalid interval layout at %u (overlap=%d span=%u offMis=%d offOver=%d)", i, j,
+                overlapsPrev, span, offsetMismatch, offsetOverruns);
+        file.close();
+        freeAll();
+        return false;
+      }
+      if (iv.first > UINT16_MAX || iv.last > UINT16_MAX || iv.offset > UINT16_MAX) {
+        canUseBmp16 = false;
+      }
+      expectedOffset += span;
+      prevLast = iv.last;
+    }
+
+    if (!file.seekSet(s.intervalsFileOffset)) {
+      LOG_ERR("SDCF", "Failed to seek back to intervals for style %u", i);
+      freeAll();
+      return false;
+    }
+
+    if (canUseBmp16) {
+      s.bmpIntervals = new (std::nothrow) PerStyle::BmpInterval16[s.header.intervalCount];
+      if (!s.bmpIntervals) {
+        LOG_ERR("SDCF", "Failed to allocate compact intervals for style %u", i);
+        freeAll();
+        return false;
+      }
       for (uint32_t j = 0; j < s.header.intervalCount; ++j) {
-        const auto& iv = s.fullIntervals[j];
-        if (iv.first > iv.last) {
-          LOG_ERR("SDCF", "Style %u: invalid interval %u (first 0x%lX > last 0x%lX)", i, j,
-                  static_cast<unsigned long>(iv.first), static_cast<unsigned long>(iv.last));
-          file.close();
+        if (file.read(reinterpret_cast<uint8_t*>(&iv), sizeof(iv)) != sizeof(iv)) {
+          LOG_ERR("SDCF", "Failed to read compact interval %u for style %u", j, i);
           freeAll();
           return false;
         }
-        const uint32_t span = iv.last - iv.first + 1;
-        const bool overlapsPrev = (j > 0 && iv.first <= prevLast);
-        const bool spanTooBig = (span > s.header.glyphCount);
-        const bool offsetMismatch = (iv.offset != expectedOffset);
-        const bool offsetOverruns = (iv.offset > s.header.glyphCount - span);
-        if (overlapsPrev || spanTooBig || offsetMismatch || offsetOverruns) {
-          LOG_ERR("SDCF", "Style %u: invalid interval layout at %u (overlap=%d span=%u offMis=%d offOver=%d)", i, j,
-                  overlapsPrev, span, offsetMismatch, offsetOverruns);
-          file.close();
-          freeAll();
-          return false;
-        }
-        expectedOffset += span;
-        prevLast = iv.last;
+        s.bmpIntervals[j] = {static_cast<uint16_t>(iv.first), static_cast<uint16_t>(iv.last),
+                             static_cast<uint16_t>(iv.offset)};
+      }
+      s.intervalsAreBmp16 = true;
+    } else {
+      s.fullIntervals = new (std::nothrow) EpdUnicodeInterval[s.header.intervalCount];
+      if (!s.fullIntervals) {
+        LOG_ERR("SDCF", "Failed to allocate %u intervals for style %u", s.header.intervalCount, i);
+        freeAll();
+        return false;
+      }
+      size_t intervalsBytes = s.header.intervalCount * sizeof(EpdUnicodeInterval);
+      if (file.read(reinterpret_cast<uint8_t*>(s.fullIntervals), intervalsBytes) != static_cast<int>(intervalsBytes)) {
+        LOG_ERR("SDCF", "Failed to read intervals for style %u", i);
+        freeAll();
+        return false;
       }
     }
 
@@ -603,13 +641,15 @@ int32_t SdCardFont::findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint)
   int right = static_cast<int>(s.header.intervalCount) - 1;
   while (left <= right) {
     int mid = left + (right - left) / 2;
-    const auto& interval = s.fullIntervals[mid];
-    if (codepoint < interval.first) {
+    const uint32_t first = s.intervalsAreBmp16 ? s.bmpIntervals[mid].first : s.fullIntervals[mid].first;
+    const uint32_t last = s.intervalsAreBmp16 ? s.bmpIntervals[mid].last : s.fullIntervals[mid].last;
+    if (codepoint < first) {
       right = mid - 1;
-    } else if (codepoint > interval.last) {
+    } else if (codepoint > last) {
       left = mid + 1;
     } else {
-      return static_cast<int32_t>(interval.offset + (codepoint - interval.first));
+      const uint32_t offset = s.intervalsAreBmp16 ? s.bmpIntervals[mid].offset : s.fullIntervals[mid].offset;
+      return static_cast<int32_t>(offset + (codepoint - first));
     }
   }
   return -1;
@@ -1257,7 +1297,7 @@ const EpdGlyph* SdCardFont::onGlyphMiss(void* ctx, uint32_t codepoint) {
 
   if (!self->loaded_ || styleIdx >= MAX_STYLES || !self->styles_[styleIdx].present) return nullptr;
   const auto& s = self->styles_[styleIdx];
-  if (!s.fullIntervals) return nullptr;
+  if (!s.fullIntervals && !s.bmpIntervals) return nullptr;
 
   // Check overflow cache first (matching both codepoint and style)
   for (uint32_t i = 0; i < self->overflowCount_; i++) {

lib/EpdFont/SdCardFont.h

@@ -58,9 +58,9 @@ class SdCardFont {
   // Returns true if advance table is populated for at least one style.
   bool hasAdvanceTable() const;
 
-  // Free mini data for all styles, restore stub EpdFontData.
-  // Also clears the temporary advance table (built per layout pass) but
-  // preserves the persistent advance cache (reused across passes).
+  // Free mini data for all styles and restore stub EpdFontData.
+  // Preserves the persistent advance cache so repeated layout passes can reuse
+  // previously fetched metrics.
   void clearCache();
 
   // Drop the persistent advance cache. Call when unloading the SD font or
@@ -140,6 +140,14 @@ class SdCardFont {
 
     // Full intervals loaded from file (kept in RAM for codepoint lookup)
     EpdUnicodeInterval* fullIntervals = nullptr;
+    struct BmpInterval16 {
+      uint16_t first;
+      uint16_t last;
+      uint16_t offset;
+    } __attribute__((packed));
+    static_assert(sizeof(BmpInterval16) == 6, "BmpInterval16 must remain compact");
+    BmpInterval16* bmpIntervals = nullptr;
+    bool intervalsAreBmp16 = false;
 
     // Persistent kern-class + ligature tables (lazy-loaded on first prewarm).
     // The full kern MATRIX is NOT resident — on Literata-class fonts a single

lib/EpdFont/SdCardFontManager.cpp

@@ -34,19 +34,15 @@ bool SdCardFontManager::loadFamily(const SdCardFontFamilyInfo& family, GfxRender
     unloadAll(renderer);
   }
 
-  // Select by ordinal position: sort available sizes, then map the font size
-  // enum (SMALL=0 .. EXTRA_LARGE=3) to the corresponding slot. When the
-  // family has fewer sizes than 4, clamp to the last available size.
-  auto sizes = family.availableSizes();
-  if (sizes.empty()) {
+  // Select the physical point size closest to the built-in reader sizes. Some
+  // CJK font packs only ship larger sizes, so ordinal selection can make
+  // MEDIUM load 18pt+ and produce oversized pages on small devices.
+  const SdCardFontFileInfo* selected = family.findClosestReaderSize(fontSizeEnum);
+  if (!selected) {
     LOG_ERR("SDMGR", "Family %s has no files to load", family.name.c_str());
     return false;
   }
 
-  uint8_t idx = fontSizeEnum;
-  if (idx >= sizes.size()) idx = sizes.size() - 1;
-  const SdCardFontFileInfo* selected = family.findFile(sizes[idx]);
-
   auto* font = new (std::nothrow) SdCardFont();
   if (!font) {
     LOG_ERR("SDMGR", "Failed to allocate SdCardFont for %s", selected->path.c_str());

lib/EpdFont/SdCardFontManager.h

@@ -15,10 +15,10 @@ class SdCardFontManager {
   SdCardFontManager(const SdCardFontManager&) = delete;
   SdCardFontManager& operator=(const SdCardFontManager&) = delete;
 
-  // Load the font file matching fontSizeEnum (SMALL=0 .. EXTRA_LARGE=3) by
-  // ordinal position in the family's sorted size list. Only one .cpfont file
-  // is loaded; other sizes remain on disk. This keeps resident interval +
-  // kern/ligature tables to one size's worth of memory.
+  // Load the font file whose physical point size is closest to the reader
+  // fontSizeEnum (SMALL=12, MEDIUM=14, LARGE=16, EXTRA_LARGE=18). Only one
+  // .cpfont file is loaded; other sizes remain on disk. This keeps resident
+  // interval + kern/ligature tables to one size's worth of memory.
   // Returns true on success.
   bool loadFamily(const SdCardFontFamilyInfo& family, GfxRenderer& renderer, uint8_t fontSizeEnum);
 
@@ -32,7 +32,7 @@ class SdCardFontManager {
   // Get name of currently loaded family (empty if none).
   const std::string& currentFamilyName() const { return loadedFamilyName_; };
 
-  // Point size that was actually loaded (closest match to targetPtSize).
+  // Point size that was actually loaded.
   // 0 if nothing loaded.
   uint8_t currentPointSize() const { return loadedPointSize_; };
 

lib/EpdFont/SdCardFontRegistry.cpp

@@ -15,6 +15,40 @@ const SdCardFontFileInfo* SdCardFontFamilyInfo::findFile(uint8_t size, uint8_t s
   return nullptr;
 }
 
+const SdCardFontFileInfo* SdCardFontFamilyInfo::findClosestReaderSize(const uint8_t fontSizeEnum,
+                                                                      const uint8_t style) const {
+  if (files.empty()) return nullptr;
+
+  uint8_t target = 14;
+  switch (fontSizeEnum) {
+    case 0:
+      target = 12;
+      break;
+    case 2:
+      target = 16;
+      break;
+    case 3:
+      target = 18;
+      break;
+    case 1:
+    default:
+      target = 14;
+      break;
+  }
+
+  const SdCardFontFileInfo* best = nullptr;
+  uint8_t bestDelta = 255;
+  for (const auto& f : files) {
+    if (f.style != style) continue;
+    const uint8_t delta = f.pointSize > target ? f.pointSize - target : target - f.pointSize;
+    if (!best || delta < bestDelta || (delta == bestDelta && f.pointSize < best->pointSize)) {
+      best = &f;
+      bestDelta = delta;
+    }
+  }
+  return best;
+}
+
 bool SdCardFontFamilyInfo::hasSize(uint8_t size) const {
   for (const auto& f : files) {
     if (f.pointSize == size) return true;

lib/EpdFont/SdCardFontRegistry.h

@@ -18,6 +18,7 @@ struct SdCardFontFamilyInfo {
   std::vector<SdCardFontFileInfo> files;
 
   const SdCardFontFileInfo* findFile(uint8_t size, uint8_t style = 0) const;
+  const SdCardFontFileInfo* findClosestReaderSize(uint8_t fontSizeEnum, uint8_t style = 0) const;
   bool hasSize(uint8_t size) const;
   std::vector<uint8_t> availableSizes() const;
 };

lib/EpdFont/scripts/fontconvert.py

@@ -248,7 +248,7 @@ unmerged_intervals = sorted(intervals + add_ints)
 intervals = []
 unvalidated_intervals = []
 for i_start, i_end in unmerged_intervals:
-    if len(unvalidated_intervals) > 0 and i_start + 1 <= unvalidated_intervals[-1][1]:
+    if len(unvalidated_intervals) > 0 and i_start <= unvalidated_intervals[-1][1] + 1:
         unvalidated_intervals[-1] = (unvalidated_intervals[-1][0], max(unvalidated_intervals[-1][1], i_end))
         continue
     unvalidated_intervals.append((i_start, i_end))