feat: add SD card font support with on-device download and web management

Add a complete SD card font subsystem that enables users to install and use custom fonts beyond the three built-in families. This combines the back-end firmware support (#1327) with the font configuration, build pipeline, CI distribution, and user-facing management UI (#1392). Core font system: - Custom .cpfont binary format (v4) with multi-style support (regular, bold, italic, bold-italic) packed into a single file per size - On-demand glyph loading from SD card with two-pass prewarm rendering to bulk-read glyphs per page, achieving near-flash performance for Latin text (~697ms vs ~681ms) and viable CJK rendering (~32% slower) - Persistent advance cache for layout measurement without SD I/O - Overflow ring buffer for glyph cache misses during rendering - Memory-conscious design: only advance tables kept in RAM; glyph bitmaps, kern tables, and ligatures loaded on demand from SD Font management: - On-device WiFi download from GitHub Releases with manifest-based discovery, install/update detection, and progress UI - Web interface font upload, listing, and deletion via /fonts page - Manual SD card copy to /fonts/ or /.fonts/ directories - Font selection integrated into Settings > Reader > Font Family Build pipeline: - Declarative YAML config (sd-fonts.yaml) as single source of truth for the 17-family font library (serif, sans, mono, accessibility) - Python converter (fontconvert_sdcard.py) for TTF/OTF to .cpfont with FreeType rasterization, class-based kerning, and ligature extraction - Parallel build orchestrator with variable font instance extraction - CI workflow publishing versioned + stable releases to a dedicated crosspoint-fonts repository with auto-incrementing revision tags - Centralized version constants (cpfont_version.py) shared across build tooling and CI, with firmware headers as manual sync points Additional fixes: - CJK characters no longer get hyphens inserted at line breaks - Advance table eliminates 30+ second stalls during CJK section indexing for paragraphs with >512 unique codepoints Closes #930 Co-authored-by: Zach Nelson <zach@zdnelson.com> Co-authored-by: Justin <itsthisjustin@users.noreply.github.com> Co-authored-by: jpirnay <jens@pirnay.com> Co-authored-by: mcrosson <kemonine@kemonine.info>
2026-05-08 21:50:06 -05:00
parent 29fd29f537
commit 7993b2bb97
57 changed files with 6064 additions and 54 deletions
@@ -0,0 +1,241 @@
+#pragma once
+
+#include <cstdint>
+
+#include "EpdFont.h"
+#include "EpdFontData.h"
+
+// On-disk binary format version for .cpfont files. Defined as a preprocessor
+// macro (rather than a constexpr) so it can be stringified into the SD-fonts
+// release URL — see FONT_MANIFEST_URL in FontDownloadActivity.h. No integer
+// suffix because stringification would include it (e.g. `4U` → `"4U"`).
+//
+// The canonical version for the build tooling lives in
+// lib/EpdFont/scripts/cpfont_version.py. This firmware-side copy must be
+// bumped manually when the firmware is updated to support a new format.
+// Reader enforcement: SdCardFont::load().
+#define CPFONT_VERSION 4
+
+class SdCardFont {
+ public:
+  static constexpr uint16_t MAX_PAGE_GLYPHS = 512;
+  static constexpr uint8_t MAX_STYLES = 4;
+
+  SdCardFont() = default;
+  ~SdCardFont();
+  // Owns raw buffers freed in dtor — no shallow-copy semantics. Make any
+  // accidental pass-by-value or move a compile-time error.
+  SdCardFont(const SdCardFont&) = delete;
+  SdCardFont& operator=(const SdCardFont&) = delete;
+  SdCardFont(SdCardFont&&) = delete;
+  SdCardFont& operator=(SdCardFont&&) = delete;
+
+  // Load .cpfont file: reads header + intervals into RAM, records file layout offsets.
+  // Supports v4 (multi-style) format.
+  // Returns true on success.
+  bool load(const char* path);
+
+  // Pre-read glyphs needed for the given UTF-8 text from SD card.
+  // styleMask: bitmask of styles to prewarm (bit 0=regular, 1=bold, 2=italic, 3=bolditalic).
+  // Default 0x0F = all present styles.
+  // When metadataOnly=true, only glyph metrics are loaded (no bitmap data).
+  // Returns number of glyphs that couldn't be loaded (0 on full success).
+  int prewarm(const char* utf8Text, uint8_t styleMask = 0x0F, bool metadataOnly = false);
+
+  // Build a compact advance-only table for layout measurement.
+  // Extracts ALL unique codepoints from utf8Text (no MAX_PAGE_GLYPHS cap),
+  // batch-reads advanceX from SD, stores in a sorted per-style table.
+  // Returns number of codepoints not found in font coverage.
+  int buildAdvanceTable(const char* utf8Text, uint8_t styleMask = 0x0F);
+
+  // Look up advanceX for a codepoint from the advance table.
+  // Returns the 12.4 fixed-point advance, or 0 if not found.
+  uint16_t getAdvance(uint32_t codepoint, uint8_t style) const;
+
+  // Returns true if advance table is populated for at least one style.
+  bool hasAdvanceTable() const;
+
+  // Free mini data for all styles, restore stub EpdFontData.
+  // Also clears the temporary advance table (built per layout pass) but
+  // preserves the persistent advance cache (reused across passes).
+  void clearCache();
+
+  // Drop the persistent advance cache. Call when unloading the SD font or
+  // when font/size/family/glyph-table state changes.
+  void clearPersistentCache();
+
+  // Returns pointer to the managed EpdFont for a given style.
+  // Returns nullptr if the style is not present.
+  EpdFont* getEpdFont(uint8_t style = 0);
+
+  // Returns true if the given style is present in this font file.
+  bool hasStyle(uint8_t style) const;
+
+  // Number of styles present in this font file.
+  uint8_t styleCount() const { return styleCount_; }
+
+  // Returns true if the glyph pointer points into the overflow buffer.
+  bool isOverflowGlyph(const EpdGlyph* glyph) const;
+
+  // Returns the bitmap for an on-demand-loaded (overflow) glyph.
+  const uint8_t* getOverflowBitmap(const EpdGlyph* glyph) const;
+
+  // Extract SdCardFont* from an opaque glyphMissCtx pointer.
+  // Used by GfxRenderer::getGlyphBitmap() to recover the SdCardFont from EpdFontData::glyphMissCtx.
+  static SdCardFont* fromMissCtx(void* ctx);
+
+  struct Stats {
+    uint32_t prewarmTotalMs = 0;
+    uint32_t sdReadTimeMs = 0;
+    uint32_t seekCount = 0;
+    uint32_t uniqueGlyphs = 0;
+    uint32_t bitmapBytes = 0;
+  };
+  void logStats(const char* label = "SDCF");
+  void resetStats();
+  const Stats& getStats() const { return stats_; }
+
+  // Content hash of the file header + style TOC entries (computed during load).
+  // Used to generate deterministic font IDs for section cache invalidation.
+  uint32_t contentHash() const { return contentHash_; }
+
+ private:
+  // Per-style metadata (parsed from file header/TOC)
+  struct CpFontHeader {
+    uint32_t intervalCount = 0;
+    uint32_t glyphCount = 0;
+    uint8_t advanceY = 0;
+    int16_t ascender = 0;
+    int16_t descender = 0;
+    bool is2Bit = false;
+    uint16_t kernLeftEntryCount = 0;
+    uint16_t kernRightEntryCount = 0;
+    uint8_t kernLeftClassCount = 0;
+    uint8_t kernRightClassCount = 0;
+    uint8_t ligaturePairCount = 0;
+  };
+
+  // All per-style data: file offsets, intervals, kern/lig, prewarm cache, EpdFont
+  struct PerStyle {
+    CpFontHeader header{};
+
+    // File layout offsets for this style's data sections
+    uint32_t intervalsFileOffset = 0;
+    uint32_t glyphsFileOffset = 0;
+    uint32_t kernLeftFileOffset = 0;
+    uint32_t kernRightFileOffset = 0;
+    uint32_t kernMatrixFileOffset = 0;
+    uint32_t ligatureFileOffset = 0;
+    uint32_t bitmapFileOffset = 0;
+
+    // Full intervals loaded from file (kept in RAM for codepoint lookup)
+    EpdUnicodeInterval* fullIntervals = nullptr;
+
+    // Persistent kern-class + ligature tables (lazy-loaded on first prewarm).
+    // The full kern MATRIX is NOT resident — on Literata-class fonts a single
+    // style's matrix is ~36-42KB contiguous, and 4 styles' worth won't fit
+    // alongside bitmaps + framebuffer on a 380KB device. Only kernLeftClasses
+    // and kernRightClasses (small codepoint→classId tables, ~3KB each) stay
+    // resident; the matrix is reconstructed per-page as miniKernMatrix.
+    EpdKernClassEntry* kernLeftClasses = nullptr;
+    EpdKernClassEntry* kernRightClasses = nullptr;
+    EpdLigaturePair* ligaturePairs = nullptr;
+    bool kernLigLoaded = false;
+
+    // Stub EpdFontData returned when not prewarmed
+    EpdFontData stubData{};
+
+    // Mini EpdFontData built during prewarm
+    EpdFontData miniData{};
+    EpdUnicodeInterval* miniIntervals = nullptr;
+    EpdGlyph* miniGlyphs = nullptr;
+    uint8_t* miniBitmap = nullptr;
+    uint32_t miniIntervalCount = 0;
+    uint32_t miniGlyphCount = 0;
+
+    // Per-page mini kern matrix (built by buildMiniKernMatrix on each full
+    // prewarm). miniKernLeftClasses/miniKernRightClasses map ONLY the codepoints
+    // used on the current page to renumbered class IDs (1..miniKern*ClassCount).
+    // miniKernMatrix is a small miniKernLeftClassCount × miniKernRightClassCount
+    // flat matrix. Typical Latin page: ~25×25 matrix = ~625 bytes per style vs
+    // ~36KB for the full Literata matrix — ~50× reduction.
+    EpdKernClassEntry* miniKernLeftClasses = nullptr;
+    EpdKernClassEntry* miniKernRightClasses = nullptr;
+    uint16_t miniKernLeftEntryCount = 0;
+    uint16_t miniKernRightEntryCount = 0;
+    uint8_t miniKernLeftClassCount = 0;
+    uint8_t miniKernRightClassCount = 0;
+    int8_t* miniKernMatrix = nullptr;
+
+    // The EpdFont whose data pointer we manage
+    EpdFont epdFont{&stubData};
+
+    bool present = false;
+  };
+
+  PerStyle styles_[MAX_STYLES] = {};
+  uint8_t styleCount_ = 0;
+
+  char filePath_[128] = {};
+
+  // Overflow context: glyphMissHandler needs to know which style it's serving
+  struct OverflowContext {
+    SdCardFont* self;
+    uint8_t styleIdx;
+  };
+  OverflowContext overflowCtx_[MAX_STYLES] = {};
+
+  // Shared on-demand overflow buffer (ring buffer of glyphs loaded via glyphMissHandler)
+  static constexpr uint32_t OVERFLOW_CAPACITY = 8;
+  struct OverflowEntry {
+    EpdGlyph glyph;
+    uint8_t* bitmap = nullptr;
+    uint32_t codepoint = 0;
+    uint8_t styleIdx = 0;
+  };
+  OverflowEntry overflow_[OVERFLOW_CAPACITY] = {};
+  uint32_t overflowCount_ = 0;
+  uint32_t overflowNext_ = 0;
+
+  // Compact advance-only table for layout measurement (per-style).
+  // Built by buildAdvanceTable(), queried by getAdvance().
+  struct AdvanceEntry {
+    uint32_t codepoint;
+    uint16_t advanceX;  // 12.4 fixed-point
+  };
+  // Per-style advance table. Sorted by codepoint for binary lookup.
+  // Bounded to ADVANCE_CACHE_LIMIT entries; persists across layout passes
+  // (across calls to clearCache()) so repeated indexing of the same font
+  // amortizes SD reads. Cleared only on font unload or clearPersistentCache().
+  static constexpr uint32_t ADVANCE_CACHE_LIMIT = 768;
+  AdvanceEntry* advanceTable_[MAX_STYLES] = {};
+  uint32_t advanceTableSize_[MAX_STYLES] = {};
+  bool advanceTableLookup(uint8_t styleIdx, uint32_t codepoint, uint16_t* outAdvance) const;
+  // Merge sortedNew (sorted by codepoint, no overlap with existing) into the
+  // advance table for styleIdx, preserving sort order; cap-truncates the tail.
+  void mergeIntoAdvanceTable(uint8_t styleIdx, const AdvanceEntry* sortedNew, uint32_t newCount);
+
+  Stats stats_;
+  uint32_t contentHash_ = 0;
+  bool loaded_ = false;
+
+  // Per-style helpers
+  void freeStyleMiniData(PerStyle& s);
+  void freeStyleAll(PerStyle& s);
+  void freeStyleKernLigatureData(PerStyle& s);
+  void freeStyleMiniKern(PerStyle& s);
+  bool loadStyleKernLigatureData(PerStyle& s);
+  bool buildMiniKernMatrix(PerStyle& s, const uint32_t* codepoints, uint32_t cpCount);
+  void applyKernLigaturePointers(PerStyle& s, EpdFontData& data) const;
+  void applyGlyphMissCallback(uint8_t styleIdx);
+  int32_t findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint) const;
+  int prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint32_t cpCount, bool metadataOnly);
+
+  // Global helpers
+  void freeAll();
+  void clearOverflow();
+  static void computeStyleFileOffsets(PerStyle& s, uint32_t baseOffset);
+
+  // Static callback for EpdFontData::glyphMissHandler (per-style via OverflowContext)
+  static const EpdGlyph* onGlyphMiss(void* ctx, uint32_t codepoint);
+};