fix: Prepare SD card font caches from txt reader (#1973)

## Summary

SD card font fixes:
- `TxtReaderActivity` needs to call `renderer.ensureSdCardFontReady` to
build the advance lookup table to support rendering with SD card fonts.
This revealed that `TxtReaderActivity` was inconsistently performing
layout with `getTextWidth`, when the renderer actually uses
`getTextAdvanceX`, which can lead to minor inconsistencies in alignment.
- Avoid allocating one big `allText` string in
`ParsedText::layoutAndExtractLines`. Instead, pass the vector of word
strings directly to `SdCardFont::buildAdvanceTable`, where the algorithm
just needs to iterate codepoints anyway.

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.

Did you use AI tools to help write this code? _**PARTIALLY**_

---------

Co-authored-by: Justin Mitchell <justin@jmitch.com>

lib/EpdFont/SdCardFont.cpp

@@ -16,11 +16,13 @@ static_assert(sizeof(EpdUnicodeInterval) == 12, "EpdUnicodeInterval must be 12 b
 static_assert(sizeof(EpdKernClassEntry) == 3, "EpdKernClassEntry must be 3 bytes to match .cpfont file layout");
 static_assert(sizeof(EpdLigaturePair) == 8, "EpdLigaturePair must be 8 bytes to match .cpfont file layout");
 
-// FNV-1a hash for content-based font ID generation
-static constexpr uint32_t FNV_OFFSET = 2166136261u;
-static constexpr uint32_t FNV_PRIME = 16777619u;
+namespace {
 
-static uint32_t fnv1a(const uint8_t* data, size_t len, uint32_t hash = FNV_OFFSET) {
+// FNV-1a hash for content-based font ID generation
+constexpr uint32_t FNV_OFFSET = 2166136261u;
+constexpr uint32_t FNV_PRIME = 16777619u;
+
+uint32_t fnv1a(const uint8_t* data, size_t len, uint32_t hash = FNV_OFFSET) {
   for (size_t i = 0; i < len; i++) {
     hash ^= data[i];
     hash *= FNV_PRIME;
@@ -29,16 +31,44 @@ static uint32_t fnv1a(const uint8_t* data, size_t len, uint32_t hash = FNV_OFFSE
 }
 
 // .cpfont magic bytes
-static constexpr char CPFONT_MAGIC[8] = {'C', 'P', 'F', 'O', 'N', 'T', '\0', '\0'};
+constexpr char CPFONT_MAGIC[8] = {'C', 'P', 'F', 'O', 'N', 'T', '\0', '\0'};
 // CPFONT_VERSION is defined as a #define in SdCardFont.h so it can be
 // stringified into FONT_MANIFEST_URL.
-static constexpr uint32_t HEADER_SIZE = 32;
-static constexpr uint32_t STYLE_TOC_ENTRY_SIZE = 32;
+constexpr uint32_t HEADER_SIZE = 32;
+constexpr uint32_t STYLE_TOC_ENTRY_SIZE = 32;
 
 // Helper to read little-endian values from byte buffer
-static inline uint16_t readU16(const uint8_t* p) { return p[0] | (p[1] << 8); }
-static inline int16_t readI16(const uint8_t* p) { return static_cast<int16_t>(p[0] | (p[1] << 8)); }
-static inline uint32_t readU32(const uint8_t* p) { return p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24); }
+inline uint16_t readU16(const uint8_t* p) { return p[0] | (p[1] << 8); }
+inline int16_t readI16(const uint8_t* p) { return static_cast<int16_t>(p[0] | (p[1] << 8)); }
+inline uint32_t readU32(const uint8_t* p) { return p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24); }
+
+// Walks a null-terminated UTF-8 string and appends each unique codepoint to
+// codepoints[0..cpCount-1] via O(n²) dedup.  Returns true if the buffer
+// reached maxCount (cap hit), false if all codepoints fit.
+bool collectUniqueCodepoints(const char* text, uint32_t* codepoints, uint32_t& cpCount, uint32_t maxCount) {
+  const unsigned char* p = reinterpret_cast<const unsigned char*>(text);
+  while (*p) {
+    uint32_t cp = utf8NextCodepoint(&p);
+    if (cp == 0) break;
+    bool found = false;
+    for (uint32_t i = 0; i < cpCount; i++) {
+      if (codepoints[i] == cp) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      if (cpCount >= maxCount) return true;
+      codepoints[cpCount++] = cp;
+    }
+  }
+  return false;
+}
+
+const char* asCStr(const std::string& s) { return s.c_str(); }
+const char* asCStr(const char* s) { return s; }
+
+}  // namespace
 
 SdCardFont::~SdCardFont() { freeAll(); }
 
@@ -1019,64 +1049,10 @@ uint16_t SdCardFont::getAdvance(uint32_t codepoint, uint8_t style) const {
   return 0;
 }
 
-int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask) {
-  if (!loaded_) return -1;
-  styleMask = resolveStyleMask(styleMask);
-  if (styleMask == 0) return 0;
-
-  // Note: advance table is preserved across calls. We only fetch codepoints
-  // not already present, then merge them in. Use clearPersistentCache() to
-  // wipe the table when the font/size/family changes.
-
-  unsigned long startMs = millis();
-
-  // Step 1: Extract unique codepoints, capped at MAX_UNIQUE_CODEPOINTS.
-  // The dedup buffer is sized to the cap, not total chars — a large EPUB section
-  // may contain 50K+ characters but real text has far fewer unique codepoints.
-  // 4096 × 4 bytes = 16KB temporary; bounded regardless of input size.
-  static constexpr uint32_t MAX_UNIQUE_CODEPOINTS = 4096;
-  uint32_t* codepoints = new (std::nothrow) uint32_t[MAX_UNIQUE_CODEPOINTS];
-  if (!codepoints) {
-    LOG_ERR("SDCF", "buildAdvanceTable: failed to allocate codepoint buffer (%u bytes)", MAX_UNIQUE_CODEPOINTS * 4);
-    return -1;
-  }
-  uint32_t cpCount = 0;
-  bool hitCap = false;
-
-  // Second pass: collect unique codepoints via O(n²) dedup.
-  // Bounded by uniqueCount × totalChars comparisons. For 2000 unique from 2291 total,
-  // worst case ~4.6M comparisons of uint32_t — ~30ms on 160MHz RISC-V, acceptable
-  // for one-time section indexing.
-  const unsigned char* p = reinterpret_cast<const unsigned char*>(utf8Text);
-  while (*p) {
-    uint32_t cp = utf8NextCodepoint(&p);
-    if (cp == 0) break;
-
-    bool found = false;
-    for (uint32_t i = 0; i < cpCount; i++) {
-      if (codepoints[i] == cp) {
-        found = true;
-        break;
-      }
-    }
-    if (!found) {
-      if (cpCount >= MAX_UNIQUE_CODEPOINTS) {
-        hitCap = true;
-        break;
-      }
-      codepoints[cpCount++] = cp;
-    }
-  }
-  if (hitCap) {
-    LOG_ERR("SDCF", "buildAdvanceTable: unique codepoint cap (%u) hit, layout may be approximate",
-            MAX_UNIQUE_CODEPOINTS);
-  }
-
-  // Sort for ordered glyph index mapping and final table output
-  std::sort(codepoints, codepoints + cpCount);
-
-  // Step 2: For each requested style, fetch any codepoints not yet cached and
-  // merge them into the persistent advance table.
+// Given a sorted array of unique codepoints, resolve glyph indices per style,
+// batch-read advanceX from SD, and merge into the persistent advance table.
+// Caller owns the codepoints buffer.
+int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask) {
   int totalMissed = 0;
   for (uint8_t si = 0; si < MAX_STYLES; si++) {
     if (!(styleMask & (1 << si)) || !styles_[si].present) continue;
@@ -1175,12 +1151,55 @@ int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask) {
             ADVANCE_CACHE_LIMIT);
   }
 
-  delete[] codepoints;
+  return totalMissed;
+}
 
+template <typename Iter>
+int SdCardFont::buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask) {
+  if (!loaded_) return -1;
+  styleMask = resolveStyleMask(styleMask);
+  if (styleMask == 0) return 0;
+
+  unsigned long startMs = millis();
+
+  // +2 reserved slots for space and hyphen injected after the main scan.
+  static constexpr uint32_t MAX_UNIQUE_CODEPOINTS = 4096;
+  uint32_t* codepoints = new (std::nothrow) uint32_t[MAX_UNIQUE_CODEPOINTS + 2];
+  if (!codepoints) {
+    LOG_ERR("SDCF", "buildAdvanceTable: failed to allocate codepoint buffer (%u bytes)", MAX_UNIQUE_CODEPOINTS * 4);
+    return -1;
+  }
+  uint32_t cpCount = 0;
+  bool hitCap = false;
+
+  for (auto it = begin; it != end && !hitCap; ++it) {
+    hitCap = collectUniqueCodepoints(asCStr(*it), codepoints, cpCount, MAX_UNIQUE_CODEPOINTS);
+  }
+
+  if (includeSpace && std::none_of(codepoints, codepoints + cpCount, [](uint32_t c) { return c == ' '; }))
+    codepoints[cpCount++] = ' ';
+  if (includeHyphen && std::none_of(codepoints, codepoints + cpCount, [](uint32_t c) { return c == '-'; }))
+    codepoints[cpCount++] = '-';
+
+  if (hitCap) {
+    LOG_ERR("SDCF", "buildAdvanceTable: unique codepoint cap (%u) hit, layout may be approximate",
+            MAX_UNIQUE_CODEPOINTS);
+  }
+  std::sort(codepoints, codepoints + cpCount);
+  int totalMissed = fetchAdvancesForCodepoints(codepoints, cpCount, styleMask);
+  delete[] codepoints;
   stats_.prewarmTotalMs = millis() - startMs;
   return totalMissed;
 }
 
+int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask) {
+  return buildAdvanceTableRange(&utf8Text, &utf8Text + 1, false, false, styleMask);
+}
+
+int SdCardFont::buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask) {
+  return buildAdvanceTableRange(words.begin(), words.end(), words.size() > 1, includeHyphen, styleMask);
+}
+
 // --- Stats ---
 
 void SdCardFont::logStats(const char* label) {

lib/EpdFont/SdCardFont.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <cstdint>
+#include <string>
+#include <vector>
 
 #include "EpdFont.h"
 #include "EpdFontData.h"
@@ -43,10 +45,11 @@ class SdCardFont {
   int prewarm(const char* utf8Text, uint8_t styleMask = 0x0F, bool metadataOnly = false);
 
   // Build a compact advance-only table for layout measurement.
-  // Extracts ALL unique codepoints from utf8Text (no MAX_PAGE_GLYPHS cap),
+  // Extracts ALL unique codepoints from words (no MAX_PAGE_GLYPHS cap),
   // batch-reads advanceX from SD, stores in a sorted per-style table.
   // Returns number of codepoints not found in font coverage.
   int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F);
+  int buildAdvanceTable(const std::vector<std::string>& words, bool includeHyphen, uint8_t styleMask = 0x0F);
 
   // Look up advanceX for a codepoint from the advance table.
   // Returns the 12.4 fixed-point advance, or 0 if not found.
@@ -236,6 +239,9 @@ class SdCardFont {
   void applyKernLigaturePointers(PerStyle& s, EpdFontData& data) const;
   void applyGlyphMissCallback(uint8_t styleIdx);
   int32_t findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint) const;
+  int fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask);
+  template <typename Iter>
+  int buildAdvanceTableRange(Iter begin, Iter end, bool includeSpace, bool includeHyphen, uint8_t styleMask);
   int prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint32_t cpCount, bool metadataOnly);
 
   // Global helpers

lib/Epub/Epub/ParsedText.cpp

@@ -255,20 +255,6 @@ void ParsedText::layoutAndExtractLines(const GfxRenderer& renderer, const int fo
   // (advanceX only, no bitmaps) for all unique codepoints in this paragraph so
   // that calculateWordWidths() can measure text without on-demand SD I/O.
   if (renderer.isSdCardFont(fontId)) {
-    // Reserve upfront so the joined text allocates exactly once. Without this,
-    // paragraphs with many words trigger a chain of vector-like reallocations
-    // inside std::string during layout — visible in prewarm timings for SD fonts.
-    size_t totalSize = hyphenationEnabled ? 1 : 0;
-    if (!words.empty()) totalSize += words.size() - 1;  // inter-word spaces
-    for (const auto& w : words) totalSize += w.size();
-    std::string allText;
-    allText.reserve(totalSize);
-    for (size_t i = 0; i < words.size(); i++) {
-      if (i > 0) allText += ' ';
-      allText += words[i];
-    }
-    if (hyphenationEnabled) allText += '-';
-
     // Style mask: only ask the SD font to load advances for styles actually
     // used in this paragraph. Style index is the low two bits (regular/bold/
     // italic/bold-italic); the underline bit is irrelevant to advance metrics.
@@ -277,7 +263,7 @@ void ParsedText::layoutAndExtractLines(const GfxRenderer& renderer, const int fo
       styleMask |= static_cast<uint8_t>(1u << (static_cast<uint8_t>(s) & 0x03));
     }
     if (styleMask == 0) styleMask = 0x01;  // defensive: regular only
-    renderer.ensureSdCardFontReady(fontId, allText.c_str(), styleMask);
+    renderer.ensureSdCardFontReady(fontId, words, hyphenationEnabled, styleMask);
   }
 
   const int pageWidth = viewportWidth;

lib/GfxRenderer/GfxRenderer.cpp

@@ -53,11 +53,22 @@ const uint8_t* GfxRenderer::getGlyphBitmap(const EpdFontData* fontData, const Ep
 
 void GfxRenderer::ensureSdCardFontReady(int fontId, const char* utf8Text, uint8_t styleMask) const {
   auto it = sdCardFonts_.find(fontId);
+  if (it != sdCardFonts_.end()) {
+    int missed = it->second->buildAdvanceTable(utf8Text, styleMask);
+    if (missed > 0) {
+      LOG_DBG("GFX", "ensureSdCardFontReady: %d glyph(s) not found", missed);
+    }
+  }
+}
+
+void GfxRenderer::ensureSdCardFontReady(int fontId, const std::vector<std::string>& words, bool includeHyphen,
+                                        uint8_t styleMask) const {
+  auto it = sdCardFonts_.find(fontId);
   if (it != sdCardFonts_.end()) {
     // Augment the persistent advance-only table for layout measurement.
     // The table survives across paragraphs/sections (capped per font), so
     // repeated indexing of the same SD font amortizes glyph-metric SD reads.
-    int missed = it->second->buildAdvanceTable(utf8Text, styleMask);
+    int missed = it->second->buildAdvanceTable(words, includeHyphen, styleMask);
     if (missed > 0) {
       LOG_DBG("GFX", "ensureSdCardFontReady: %d glyph(s) not found", missed);
     }

lib/GfxRenderer/GfxRenderer.h

@@ -94,6 +94,8 @@ class GfxRenderer {
   // (which holds a const GfxRenderer&) before measuring word widths. Safe to call on non-SD fonts (no-op).
   // styleMask: bitmask of styles to prepare (bit 0=regular, 1=bold, 2=italic, 3=bold-italic).
   void ensureSdCardFontReady(int fontId, const char* utf8Text, uint8_t styleMask = 0x0F) const;
+  void ensureSdCardFontReady(int fontId, const std::vector<std::string>& words, bool includeHyphen,
+                             uint8_t styleMask = 0x0F) const;
 
   // Orientation control (affects logical width/height and coordinate transforms)
   void setOrientation(const Orientation o) { orientation = o; }

src/activities/reader/TxtReaderActivity.cpp

@@ -192,6 +192,17 @@ bool TxtReaderActivity::loadPageAtOffset(size_t offset, std::vector<std::string>
   }
   buffer[chunkSize] = '\0';
 
+  // Prime the SD card font's advance table with this chunk's codepoints.
+  // Without this, every getTextAdvanceX() call in the wrap loop below triggers
+  // on-demand glyph loads through the 8-slot overflow ring buffer, which
+  // thrashes for any text with more than 8 unique chars (i.e. all English),
+  // floods the heap with short-lived bitmap allocations, and eventually
+  // corrupts FreeRTOS state. The advance table persists across calls per
+  // font, so the cost amortizes to ~ASCII-size after the first chunk.
+  if (renderer.isSdCardFont(cachedFontId)) {
+    renderer.ensureSdCardFontReady(cachedFontId, reinterpret_cast<const char*>(buffer), /*styleMask=*/0x01);
+  }
+
   // Parse lines from buffer
   size_t pos = 0;
 
@@ -231,7 +242,7 @@ bool TxtReaderActivity::loadPageAtOffset(size_t offset, std::vector<std::string>
         break;
       }
 
-      int lineWidth = renderer.getTextWidth(cachedFontId, line.c_str());
+      int lineWidth = renderer.getTextAdvanceX(cachedFontId, line.c_str(), EpdFontFamily::REGULAR);
 
       if (lineWidth <= viewportWidth) {
         outLines.push_back(line);
@@ -242,7 +253,8 @@ bool TxtReaderActivity::loadPageAtOffset(size_t offset, std::vector<std::string>
 
       // Find break point
       size_t breakPos = line.length();
-      while (breakPos > 0 && renderer.getTextWidth(cachedFontId, line.substr(0, breakPos).c_str()) > viewportWidth) {
+      while (breakPos > 0 && renderer.getTextAdvanceX(cachedFontId, line.substr(0, breakPos).c_str(),
+                                                      EpdFontFamily::REGULAR) > viewportWidth) {
         // Try to break at space
         size_t spacePos = line.rfind(' ', breakPos - 1);
         if (spacePos != std::string::npos && spacePos > 0) {
@@ -354,12 +366,12 @@ void TxtReaderActivity::renderPage() {
             // x already set to left margin
             break;
           case CrossPointSettings::CENTER_ALIGN: {
-            int textWidth = renderer.getTextWidth(cachedFontId, line.c_str());
+            int textWidth = renderer.getTextAdvanceX(cachedFontId, line.c_str(), EpdFontFamily::REGULAR);
             x = cachedOrientedMarginLeft + (contentWidth - textWidth) / 2;
             break;
           }
           case CrossPointSettings::RIGHT_ALIGN: {
-            int textWidth = renderer.getTextWidth(cachedFontId, line.c_str());
+            int textWidth = renderer.getTextAdvanceX(cachedFontId, line.c_str(), EpdFontFamily::REGULAR);
             x = cachedOrientedMarginLeft + contentWidth - textWidth;
             break;
           }