perf: font-compression improvements (#1056)

## Purpose This PR includes some preparatory changes that are needed for an upcoming performant CJK font feature. The changes have no impact on render time and heap allocation for latin text. **Despite this, I think these changes stand on their own as a better font compression/decompression implementation.** ## Summary - Font decompressor rewrite: Replaced the 4-slot LRU group cache with a two-tier system — a page buffer (glyphs prewarmed before rendering begins) and a hot-group fallback (last decompressed group retained for non-prewarmed glyphs). - Byte-aligned compressed bitmap format: Glyph bitmaps within compressed groups are now stored row-padded rather than tightly packed before DEFLATE compression, improving compression ratios by making identical pixel rows produce identical byte patterns. Glyphs are compacted back to packed format on demand at render time. Reduces flash size by 155 KB. - Page prewarm system: Added `Page::collectText` and `Page::getDominantStyle` to extract per-style glyph requirements before rendering, and `GfxRenderer::prewarmFontCache` to pre-decompress only the groups needed for the dominant style — eliminating mid-render decompression for the common case. - UTF-8 robustness fixes: `utf8NextCodepoint` now validates continuation bytes and returns a replacement glyph on malformed input; `ChapterHtmlSlimParser` correctly preserves incomplete multi-byte sequences across word-buffer flush boundaries rather than splitting them. --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**YES**_ Architecture and design was done by me, refined a bit by Claude. Code mostly by Claude, but not entirely.
2026-03-11 21:05:46 +01:00
parent b467ea7973
commit f1e9dc7f30
70 changed files with 104437 additions and 120058 deletions
@@ -50,7 +50,7 @@ typedef struct {
  uint32_t compressedSize;    ///< Compressed DEFLATE stream size
  uint32_t uncompressedSize;  ///< Decompressed size
  uint16_t glyphCount;        ///< Number of glyphs in this group
-  uint16_t firstGlyphIndex;   ///< First glyph index in the global glyph array
+  uint32_t firstGlyphIndex;   ///< First glyph index in the global glyph array
 } EpdFontGroup;

 /// Glyph interval structure
@@ -86,6 +86,7 @@ typedef struct {
  bool is2Bit;
  const EpdFontGroup* groups;                 ///< NULL for uncompressed fonts
  uint16_t groupCount;                        ///< 0 for uncompressed fonts
+  const uint16_t* glyphToGroup;               ///< Per-glyph group ID (nullptr for contiguous-group fonts)
  const EpdKernClassEntry* kernLeftClasses;   ///< Sorted left-side class map (nullptr if none)
  const EpdKernClassEntry* kernRightClasses;  ///< Sorted right-side class map (nullptr if none)
  const int8_t* kernMatrix;              ///< Flat leftClassCount x rightClassCount matrix, 4.4 fixed-point in pixels
@@ -1,34 +1,55 @@
 #include "FontDecompressor.h"

+#include <Arduino.h>
 #include <Logging.h>
+#include <Utf8.h>

 #include <cstdlib>

+FontDecompressor::~FontDecompressor() { deinit(); }
+
 bool FontDecompressor::init() {
  clearCache();
  return true;
 }

-void FontDecompressor::freeAllEntries() {
-  for (auto& entry : cache) {
-    if (entry.data) {
-      free(entry.data);
-      entry.data = nullptr;
-    }
-    entry.valid = false;
-  }
+void FontDecompressor::deinit() {
+  freePageBuffer();
+  freeHotGroup();
 }

-void FontDecompressor::deinit() { freeAllEntries(); }
-
 void FontDecompressor::clearCache() {
-  freeAllEntries();
-  accessCounter = 0;
+  freePageBuffer();
+  freeHotGroup();
 }

-uint16_t FontDecompressor::getGroupIndex(const EpdFontData* fontData, uint16_t glyphIndex) {
+void FontDecompressor::freePageBuffer() {
+  free(pageBuffer);
+  pageBuffer = nullptr;
+  free(pageGlyphs);
+  pageGlyphs = nullptr;
+  pageFont = nullptr;
+  pageGlyphCount = 0;
+}
+
+void FontDecompressor::freeHotGroup() {
+  hotGroup.clear();
+  hotGroup.shrink_to_fit();
+  hotGroupFont = nullptr;
+  hotGroupIndex = UINT16_MAX;
+  hotGlyphBuf.clear();
+  hotGlyphBuf.shrink_to_fit();
+}
+
+uint16_t FontDecompressor::getGroupIndex(const EpdFontData* fontData, uint32_t glyphIndex) {
+  // O(1) path for frequency-grouped fonts with glyphToGroup mapping
+  if (fontData->glyphToGroup != nullptr) {
+    return fontData->glyphToGroup[glyphIndex];
+  }
+
+  // Contiguous-group fonts: linear scan
  for (uint16_t i = 0; i < fontData->groupCount; i++) {
-    uint16_t first = fontData->groups[i].firstGlyphIndex;
+    uint32_t first = fontData->groups[i].firstGlyphIndex;
    if (glyphIndex >= first && glyphIndex < first + fontData->groups[i].glyphCount) {
      return i;
    }
@@ -36,99 +57,412 @@ uint16_t FontDecompressor::getGroupIndex(const EpdFontData* fontData, uint16_t g
  return fontData->groupCount;  // sentinel = not found
 }

-FontDecompressor::CacheEntry* FontDecompressor::findInCache(const EpdFontData* fontData, uint16_t groupIndex) {
-  for (auto& entry : cache) {
-    if (entry.valid && entry.font == fontData && entry.groupIndex == groupIndex) {
-      return &entry;
-    }
-  }
-  return nullptr;
-}
-
-FontDecompressor::CacheEntry* FontDecompressor::findEvictionCandidate() {
-  // Find an invalid slot first
-  for (auto& entry : cache) {
-    if (!entry.valid) {
-      return &entry;
-    }
-  }
-  // Otherwise evict LRU
-  CacheEntry* lru = &cache[0];
-  for (auto& entry : cache) {
-    if (entry.lastUsed < lru->lastUsed) {
-      lru = &entry;
-    }
-  }
-  return lru;
-}
-
-bool FontDecompressor::decompressGroup(const EpdFontData* fontData, uint16_t groupIndex, CacheEntry* entry) {
+bool FontDecompressor::decompressGroup(const EpdFontData* fontData, uint16_t groupIndex, uint8_t* outBuf,
+                                       uint32_t outSize) {
  const EpdFontGroup& group = fontData->groups[groupIndex];

-  // Free old buffer if reusing a slot
-  if (entry->data) {
-    free(entry->data);
-    entry->data = nullptr;
-  }
-  entry->valid = false;
-
-  // Allocate output buffer
-  auto* outBuf = static_cast<uint8_t*>(malloc(group.uncompressedSize));
-  if (!outBuf) {
-    LOG_ERR("FDC", "Failed to allocate %u bytes for group %u", group.uncompressedSize, groupIndex);
-    return false;
-  }
-
+  const uint32_t tDecomp = millis();
  inflateReader.init(false);
  inflateReader.setSource(&fontData->bitmap[group.compressedOffset], group.compressedSize);
-  if (!inflateReader.read(outBuf, group.uncompressedSize)) {
+  if (!inflateReader.read(outBuf, outSize)) {
+    stats.decompressTimeMs += millis() - tDecomp;
    LOG_ERR("FDC", "Decompression failed for group %u", groupIndex);
-    free(outBuf);
    return false;
  }
-
-  entry->font = fontData;
-  entry->groupIndex = groupIndex;
-  entry->data = outBuf;
-  entry->dataSize = group.uncompressedSize;
-  entry->valid = true;
+  stats.decompressTimeMs += millis() - tDecomp;
  return true;
 }

-const uint8_t* FontDecompressor::getBitmap(const EpdFontData* fontData, const EpdGlyph* glyph, uint16_t glyphIndex) {
+// --- Byte-aligned helpers ---
+
+uint32_t FontDecompressor::getAlignedOffset(const EpdFontData* fontData, uint16_t groupIndex, uint32_t glyphIndex) {
+  uint32_t offset = 0;
+
+  auto accumGlyph = [&](const EpdGlyph& g) {
+    if (g.width > 0 && g.height > 0) {
+      offset += ((g.width + 3) / 4) * g.height;
+    }
+  };
+
+  if (fontData->glyphToGroup) {
+    // Frequency-grouped: scan glyphs before glyphIndex that belong to this group
+    for (uint32_t i = 0; i < glyphIndex; i++) {
+      if (fontData->glyphToGroup[i] == groupIndex) {
+        accumGlyph(fontData->glyph[i]);
+      }
+    }
+  } else {
+    // Contiguous-group: sum aligned sizes of preceding glyphs in the group
+    const EpdFontGroup& group = fontData->groups[groupIndex];
+    for (uint32_t i = group.firstGlyphIndex; i < glyphIndex; i++) {
+      accumGlyph(fontData->glyph[i]);
+    }
+  }
+
+  return offset;
+}
+
+void FontDecompressor::compactSingleGlyph(const uint8_t* alignedSrc, uint8_t* packedDst, uint8_t width,
+                                          uint8_t height) {
+  if (width == 0 || height == 0) return;
+  const uint32_t rowStride = (width + 3) / 4;
+  if (width % 4 == 0) {
+    memcpy(packedDst, alignedSrc, rowStride * height);
+    return;
+  }
+  uint8_t outByte = 0, outBits = 0;
+  uint32_t writeIdx = 0;
+  for (uint8_t y = 0; y < height; y++) {
+    for (uint8_t x = 0; x < width; x++) {
+      outByte = (outByte << 2) | ((alignedSrc[y * rowStride + x / 4] >> ((3 - (x % 4)) * 2)) & 0x3);
+      outBits += 2;
+      if (outBits == 8) {
+        packedDst[writeIdx++] = outByte;
+        outByte = 0;
+        outBits = 0;
+      }
+    }
+  }
+  if (outBits > 0) packedDst[writeIdx] = outByte << (8 - outBits);
+}
+
+// --- getBitmap: page buffer → hot group → decompress ---
+
+const uint8_t* FontDecompressor::getBitmap(const EpdFontData* fontData, const EpdGlyph* glyph, uint32_t glyphIndex) {
+  const uint32_t tStart = micros();
+  stats.getBitmapCalls++;
+
  if (!fontData->groups || fontData->groupCount == 0) {
+    stats.getBitmapTimeUs += micros() - tStart;
    return &fontData->bitmap[glyph->dataOffset];
  }

+  // Check page buffer first (populated by prewarmCache)
+  if (pageBuffer && pageFont == fontData && pageGlyphCount > 0) {
+    int left = 0, right = pageGlyphCount - 1;
+    while (left <= right) {
+      int mid = left + (right - left) / 2;
+      if (pageGlyphs[mid].glyphIndex == glyphIndex) {
+        if (pageGlyphs[mid].bufferOffset != UINT32_MAX) {
+          stats.cacheHits++;
+          stats.getBitmapTimeUs += micros() - tStart;
+          return &pageBuffer[pageGlyphs[mid].bufferOffset];
+        }
+        break;  // Not extracted during prewarm; fall through to hot-group path
+      }
+      if (pageGlyphs[mid].glyphIndex < glyphIndex)
+        left = mid + 1;
+      else
+        right = mid - 1;
+    }
+  }
+
+  // Fallback: hot group slot
  uint16_t groupIndex = getGroupIndex(fontData, glyphIndex);
  if (groupIndex >= fontData->groupCount) {
    LOG_ERR("FDC", "Glyph %u not found in any group", glyphIndex);
+    stats.getBitmapTimeUs += micros() - tStart;
    return nullptr;
  }

-  // Check cache
-  CacheEntry* entry = findInCache(fontData, groupIndex);
-  if (entry) {
-    entry->lastUsed = ++accessCounter;
-    if (glyph->dataOffset + glyph->dataLength > entry->dataSize) {
-      LOG_ERR("FDC", "dataOffset %u + dataLength %u out of bounds for group %u (size %u)", glyph->dataOffset,
-              glyph->dataLength, groupIndex, entry->dataSize);
+  // Check if hot group already has this group decompressed — if not, decompress it
+  if (!(!hotGroup.empty() && hotGroupFont == fontData && hotGroupIndex == groupIndex)) {
+    stats.cacheMisses++;
+    const EpdFontGroup& group = fontData->groups[groupIndex];
+
+    hotGroup.resize(group.uncompressedSize);
+    if (hotGroup.empty()) {
+      LOG_ERR("FDC", "Failed to allocate %u bytes for hot group %u", group.uncompressedSize, groupIndex);
+      hotGroupFont = nullptr;
+      hotGroupIndex = UINT16_MAX;
+      stats.getBitmapTimeUs += micros() - tStart;
      return nullptr;
    }
-    return &entry->data[glyph->dataOffset];
+
+    if (!decompressGroup(fontData, groupIndex, hotGroup.data(), group.uncompressedSize)) {
+      hotGroup.clear();
+      hotGroup.shrink_to_fit();
+      hotGroupFont = nullptr;
+      hotGroupIndex = UINT16_MAX;
+      stats.getBitmapTimeUs += micros() - tStart;
+      return nullptr;
+    }
+
+    hotGroupFont = fontData;
+    hotGroupIndex = groupIndex;
+    stats.hotGroupBytes = group.uncompressedSize;
+  } else {
+    stats.cacheHits++;
  }

-  // Cache miss - decompress
-  entry = findEvictionCandidate();
-  if (!decompressGroup(fontData, groupIndex, entry)) {
+  // Compact just the requested glyph from byte-aligned data into scratch buffer
+  if (glyph->dataLength > hotGlyphBuf.size()) {
+    hotGlyphBuf.resize(glyph->dataLength);
+  }
+  if (hotGlyphBuf.empty()) {
+    stats.getBitmapTimeUs += micros() - tStart;
    return nullptr;
  }

-  entry->lastUsed = ++accessCounter;
-  if (glyph->dataOffset + glyph->dataLength > entry->dataSize) {
-    LOG_ERR("FDC", "dataOffset %u + dataLength %u out of bounds for group %u (size %u)", glyph->dataOffset,
-            glyph->dataLength, groupIndex, entry->dataSize);
-    return nullptr;
+  uint32_t alignedOff = getAlignedOffset(fontData, groupIndex, glyphIndex);
+  compactSingleGlyph(&hotGroup[alignedOff], hotGlyphBuf.data(), glyph->width, glyph->height);
+  stats.getBitmapTimeUs += micros() - tStart;
+  return hotGlyphBuf.data();
+}
+
+// --- Prewarm: pre-decompress glyph bitmaps for a page of text ---
+
+int32_t FontDecompressor::findGlyphIndex(const EpdFontData* fontData, uint32_t codepoint) {
+  const EpdUnicodeInterval* intervals = fontData->intervals;
+  const int count = fontData->intervalCount;
+
+  if (count == 0) return -1;
+
+  // Binary search
+  int left = 0;
+  int right = count - 1;
+
+  while (left <= right) {
+    const int mid = left + (right - left) / 2;
+    const EpdUnicodeInterval* interval = &intervals[mid];
+
+    if (codepoint < interval->first) {
+      right = mid - 1;
+    } else if (codepoint > interval->last) {
+      left = mid + 1;
+    } else {
+      return static_cast<int32_t>(interval->offset + (codepoint - interval->first));
+    }
  }
-  return &entry->data[glyph->dataOffset];
+
+  return -1;
+}
+
+int FontDecompressor::prewarmCache(const EpdFontData* fontData, const char* utf8Text) {
+  freePageBuffer();
+  if (!fontData || !fontData->groups || !utf8Text) return 0;
+
+  // Step 1: Collect unique glyph indices needed for this page
+  uint32_t neededGlyphs[MAX_PAGE_GLYPHS];
+  uint16_t glyphCount = 0;
+  bool glyphCapWarned = false;
+
+  const unsigned char* p = reinterpret_cast<const unsigned char*>(utf8Text);
+  while (*p) {
+    uint32_t cp = utf8NextCodepoint(&p);
+    if (cp == 0) break;
+
+    int32_t glyphIdx = findGlyphIndex(fontData, cp);
+    if (glyphIdx < 0) continue;
+
+    // Deduplicate
+    bool found = false;
+    for (uint16_t i = 0; i < glyphCount; i++) {
+      if (neededGlyphs[i] == static_cast<uint32_t>(glyphIdx)) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      if (glyphCount < MAX_PAGE_GLYPHS) {
+        neededGlyphs[glyphCount++] = static_cast<uint32_t>(glyphIdx);
+      } else if (!glyphCapWarned) {
+        LOG_DBG("FDC", "Glyph cap (%u) reached during prewarm; excess glyphs will use hot-group fallback",
+                MAX_PAGE_GLYPHS);
+        glyphCapWarned = true;
+      }
+    }
+  }
+
+  if (glyphCount == 0) return 0;
+
+  // Step 2: Compute total buffer size and collect unique groups
+  uint32_t totalBytes = 0;
+  uint16_t neededGroups[128];
+  uint8_t groupCount = 0;
+  bool groupCapWarned = false;
+
+  for (uint16_t i = 0; i < glyphCount; i++) {
+    totalBytes += fontData->glyph[neededGlyphs[i]].dataLength;
+    uint16_t gi = getGroupIndex(fontData, neededGlyphs[i]);
+    bool found = false;
+    for (uint8_t j = 0; j < groupCount; j++) {
+      if (neededGroups[j] == gi) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      if (groupCount < 128) {
+        neededGroups[groupCount++] = gi;
+      } else if (!groupCapWarned) {
+        LOG_DBG("FDC", "Group cap (128) reached during prewarm; some groups will use hot-group fallback");
+        groupCapWarned = true;
+      }
+    }
+  }
+
+  stats.uniqueGroupsAccessed = groupCount;
+
+  // Step 3: Allocate page buffer and lookup table
+  pageBuffer = static_cast<uint8_t*>(malloc(totalBytes));
+  pageGlyphs = static_cast<PageGlyphEntry*>(malloc(glyphCount * sizeof(PageGlyphEntry)));
+  if (!pageBuffer || !pageGlyphs) {
+    LOG_ERR("FDC", "Failed to allocate page buffer (%u bytes, %u glyphs)", totalBytes, glyphCount);
+    freePageBuffer();
+    return glyphCount;
+  }
+  stats.pageBufferBytes = totalBytes;
+  stats.pageGlyphsBytes = glyphCount * sizeof(PageGlyphEntry);
+
+  pageFont = fontData;
+  pageGlyphCount = glyphCount;
+
+  // Initialize lookup entries (bufferOffset = UINT32_MAX means not yet extracted)
+  for (uint16_t i = 0; i < glyphCount; i++) {
+    pageGlyphs[i] = {neededGlyphs[i], UINT32_MAX, 0};
+  }
+
+  // Sort by glyphIndex for binary search in getBitmap()
+  for (uint16_t i = 1; i < glyphCount; i++) {
+    PageGlyphEntry key = pageGlyphs[i];
+    int j = i - 1;
+    while (j >= 0 && pageGlyphs[j].glyphIndex > key.glyphIndex) {
+      pageGlyphs[j + 1] = pageGlyphs[j];
+      j--;
+    }
+    pageGlyphs[j + 1] = key;
+  }
+
+  // Step 3b: Pre-scan to compute each needed glyph's byte-aligned offset within its group.
+  // This avoids recomputing aligned offsets per group during extraction in step 4.
+  uint32_t groupAlignedTracker[128] = {};  // running byte-aligned offset for each needed group
+
+  if (fontData->glyphToGroup) {
+    // Frequency-grouped: single O(totalGlyphs) pass through glyphToGroup
+    const auto& lastInterval = fontData->intervals[fontData->intervalCount - 1];
+    const uint32_t totalGlyphs = lastInterval.offset + (lastInterval.last - lastInterval.first + 1);
+
+    for (uint32_t i = 0; i < totalGlyphs; i++) {
+      const uint16_t gi = fontData->glyphToGroup[i];
+      // Find this glyph's group position in neededGroups
+      uint8_t gpPos = groupCount;
+      for (uint8_t j = 0; j < groupCount; j++) {
+        if (neededGroups[j] == gi) {
+          gpPos = j;
+          break;
+        }
+      }
+      if (gpPos == groupCount) continue;  // not a needed group
+
+      const EpdGlyph& glyph = fontData->glyph[i];
+
+      // Binary search in sorted pageGlyphs to find if glyph i is needed
+      int left = 0, right = (int)pageGlyphCount - 1;
+      while (left <= right) {
+        const int mid = left + (right - left) / 2;
+        if (pageGlyphs[mid].glyphIndex == i) {
+          pageGlyphs[mid].alignedOffset = groupAlignedTracker[gpPos];
+          break;
+        }
+        if (pageGlyphs[mid].glyphIndex < i)
+          left = mid + 1;
+        else
+          right = mid - 1;
+      }
+
+      if (glyph.width > 0 && glyph.height > 0) {
+        groupAlignedTracker[gpPos] += ((glyph.width + 3) / 4) * glyph.height;
+      }
+    }
+  } else {
+    // Contiguous-group: iterate each needed group's glyphs directly
+    for (uint8_t g = 0; g < groupCount; g++) {
+      const EpdFontGroup& group = fontData->groups[neededGroups[g]];
+      uint32_t alignedOff = 0;
+      for (uint16_t j = 0; j < group.glyphCount; j++) {
+        const uint32_t glyphI = group.firstGlyphIndex + j;
+        const EpdGlyph& glyph = fontData->glyph[glyphI];
+
+        int left = 0, right = (int)pageGlyphCount - 1;
+        while (left <= right) {
+          const int mid = left + (right - left) / 2;
+          if (pageGlyphs[mid].glyphIndex == glyphI) {
+            pageGlyphs[mid].alignedOffset = alignedOff;
+            break;
+          }
+          if (pageGlyphs[mid].glyphIndex < glyphI)
+            left = mid + 1;
+          else
+            right = mid - 1;
+        }
+
+        if (glyph.width > 0 && glyph.height > 0) {
+          alignedOff += ((glyph.width + 3) / 4) * glyph.height;
+        }
+      }
+    }
+  }
+
+  // Step 4: For each unique group, decompress to temp buffer and extract needed glyphs
+  uint32_t writeOffset = 0;
+  int missed = 0;
+
+  for (uint8_t g = 0; g < groupCount; g++) {
+    uint16_t groupIdx = neededGroups[g];
+    const EpdFontGroup& group = fontData->groups[groupIdx];
+
+    auto* tempBuf = static_cast<uint8_t*>(malloc(group.uncompressedSize));
+    if (!tempBuf) {
+      LOG_ERR("FDC", "Failed to allocate temp buffer (%u bytes) for group %u", group.uncompressedSize, groupIdx);
+      missed++;
+      continue;
+    }
+    if (group.uncompressedSize > stats.peakTempBytes) {
+      stats.peakTempBytes = group.uncompressedSize;
+    }
+
+    if (!decompressGroup(fontData, groupIdx, tempBuf, group.uncompressedSize)) {
+      free(tempBuf);
+      missed++;
+      continue;
+    }
+
+    // Extract needed glyphs directly from the byte-aligned temp buffer, compacting on the fly.
+    // alignedOffset was pre-computed in step 3b — no full-group compact scan needed.
+    for (uint16_t i = 0; i < pageGlyphCount; i++) {
+      if (pageGlyphs[i].bufferOffset != UINT32_MAX) continue;  // already extracted
+      if (getGroupIndex(fontData, pageGlyphs[i].glyphIndex) != groupIdx) continue;
+
+      const EpdGlyph& glyph = fontData->glyph[pageGlyphs[i].glyphIndex];
+      compactSingleGlyph(&tempBuf[pageGlyphs[i].alignedOffset], &pageBuffer[writeOffset], glyph.width, glyph.height);
+      pageGlyphs[i].bufferOffset = writeOffset;
+      writeOffset += glyph.dataLength;
+    }
+
+    free(tempBuf);
+  }
+
+  LOG_DBG("FDC", "Prewarm: %u glyphs in %u bytes from %u groups (%d missed)", glyphCount, writeOffset, groupCount,
+          missed);
+
+  return missed;
+}
+
+// --- Stats ---
+
+void FontDecompressor::resetStats() { stats = Stats{}; }
+
+void FontDecompressor::logStats(const char* label) {
+  const uint32_t total = stats.cacheHits + stats.cacheMisses;
+  LOG_DBG("FDC", "[%s] hits=%lu misses=%lu (%.1f%% hit rate)", label, stats.cacheHits, stats.cacheMisses,
+          total > 0 ? 100.0f * stats.cacheHits / total : 0.0f);
+  LOG_DBG("FDC", "[%s] decompress=%lums groups_accessed=%u", label, stats.decompressTimeMs, stats.uniqueGroupsAccessed);
+  LOG_DBG("FDC", "[%s] mem: pageBuf=%lu pageGlyphs=%lu hotGroup=%lu peakTemp=%lu", label, stats.pageBufferBytes,
+          stats.pageGlyphsBytes, stats.hotGroupBytes, stats.peakTempBytes);
+  if (stats.getBitmapCalls > 0) {
+    LOG_DBG("FDC", "[%s] getBitmap: %lu calls, %luus total, %luus/call avg", label, stats.getBitmapCalls,
+            stats.getBitmapTimeUs, stats.getBitmapTimeUs / stats.getBitmapCalls);
+  }
+  resetStats();
 }
@@ -2,39 +2,78 @@

 #include <InflateReader.h>

+#include <vector>
+
 #include "EpdFontData.h"

 class FontDecompressor {
 public:
+  static constexpr uint16_t MAX_PAGE_GLYPHS = 512;
+
+  FontDecompressor() = default;
+  ~FontDecompressor();
+
  bool init();
  void deinit();

  // Returns pointer to decompressed bitmap data for the given glyph.
-  // Valid until LRU eviction (safe for the duration of one glyph render).
-  const uint8_t* getBitmap(const EpdFontData* fontData, const EpdGlyph* glyph, uint16_t glyphIndex);
+  // Checks the page buffer (from prewarm) first, then falls back to the hot group slot.
+  const uint8_t* getBitmap(const EpdFontData* fontData, const EpdGlyph* glyph, uint32_t glyphIndex);

-  // Evict all cached decompressed groups (call between pages for within-page-only caching).
+  // Free all cached data (page buffer + hot group).
  void clearCache();

- private:
-  static constexpr uint8_t CACHE_SLOTS = 4;
+  // Pre-scan UTF-8 text and extract needed glyph bitmaps into a flat page buffer.
+  // Each group is decompressed once into a temp buffer; only needed glyphs are kept.
+  // Returns the number of glyphs that couldn't be loaded (0 on full success).
+  int prewarmCache(const EpdFontData* fontData, const char* utf8Text);

-  struct CacheEntry {
-    const EpdFontData* font = nullptr;
-    uint16_t groupIndex = 0;
-    uint8_t* data = nullptr;
-    uint32_t dataSize = 0;
-    uint32_t lastUsed = 0;
-    bool valid = false;
+  struct Stats {
+    uint32_t cacheHits = 0;
+    uint32_t cacheMisses = 0;
+    uint32_t decompressTimeMs = 0;
+    uint16_t uniqueGroupsAccessed = 0;
+    uint32_t pageBufferBytes = 0;  // pageBuffer allocation
+    uint32_t pageGlyphsBytes = 0;  // pageGlyphs lookup table allocation
+    uint32_t hotGroupBytes = 0;    // current hot group allocation
+    uint32_t peakTempBytes = 0;    // largest temp buffer in prewarm
+    uint32_t getBitmapTimeUs = 0;  // cumulative getBitmap time (micros)
+    uint32_t getBitmapCalls = 0;   // number of getBitmap calls
  };
+  void logStats(const char* label = "FDC");
+  void resetStats();
+  const Stats& getStats() const { return stats; }

+ private:
+  Stats stats;
  InflateReader inflateReader;
-  CacheEntry cache[CACHE_SLOTS] = {};
-  uint32_t accessCounter = 0;

-  void freeAllEntries();
-  uint16_t getGroupIndex(const EpdFontData* fontData, uint16_t glyphIndex);
-  CacheEntry* findInCache(const EpdFontData* fontData, uint16_t groupIndex);
-  CacheEntry* findEvictionCandidate();
-  bool decompressGroup(const EpdFontData* fontData, uint16_t groupIndex, CacheEntry* entry);
+  // Page buffer: flat array of prewarmed glyph bitmaps with sorted lookup
+  struct PageGlyphEntry {
+    uint32_t glyphIndex;
+    uint32_t bufferOffset;
+    uint32_t alignedOffset;  // byte-aligned offset within its decompressed group (set during prewarm pre-scan)
+  };
+  uint8_t* pageBuffer = nullptr;
+  const EpdFontData* pageFont = nullptr;
+  PageGlyphEntry* pageGlyphs = nullptr;
+  uint16_t pageGlyphCount = 0;
+
+  // Hot group: last decompressed group (byte-aligned) for non-prewarmed fallback path.
+  // Kept in byte-aligned format; individual glyphs are compacted on demand into hotGlyphBuf.
+  const EpdFontData* hotGroupFont = nullptr;
+  uint16_t hotGroupIndex = UINT16_MAX;
+  std::vector<uint8_t> hotGroup;
+
+  // Scratch buffer for compacting a single glyph from the hot group.
+  // Valid until the next getBitmap() call.
+  std::vector<uint8_t> hotGlyphBuf;
+
+  void freePageBuffer();
+  void freeHotGroup();
+  uint16_t getGroupIndex(const EpdFontData* fontData, uint32_t glyphIndex);
+  uint32_t getAlignedOffset(const EpdFontData* fontData, uint16_t groupIndex, uint32_t glyphIndex);
+  bool decompressGroup(const EpdFontData* fontData, uint16_t groupIndex, uint8_t* outBuf, uint32_t outSize);
+  static void compactSingleGlyph(const uint8_t* alignedSrc, uint8_t* packedDst, uint8_t width, uint8_t height);
+  static int32_t findGlyphIndex(const EpdFontData* fontData, uint32_t codepoint);
 };
@@ -3118,6 +3118,7 @@ static const EpdFontData notosans_8_regular = {
    false,
    nullptr,
    0,
+    nullptr,
    notosans_8_regularKernLeftClasses,
    notosans_8_regularKernRightClasses,
    notosans_8_regularKernMatrix,
@@ -3092,6 +3092,7 @@ static const EpdFontData ubuntu_10_bold = {
    false,
    nullptr,
    0,
+    nullptr,
    ubuntu_10_boldKernLeftClasses,
    ubuntu_10_boldKernRightClasses,
    ubuntu_10_boldKernMatrix,
@@ -2768,6 +2768,7 @@ static const EpdFontData ubuntu_10_regular = {
    false,
    nullptr,
    0,
+    nullptr,
    ubuntu_10_regularKernLeftClasses,
    ubuntu_10_regularKernRightClasses,
    ubuntu_10_regularKernMatrix,
@@ -3478,6 +3478,7 @@ static const EpdFontData ubuntu_12_bold = {
    false,
    nullptr,
    0,
+    nullptr,
    ubuntu_12_boldKernLeftClasses,
    ubuntu_12_boldKernRightClasses,
    ubuntu_12_boldKernMatrix,
@@ -3129,6 +3129,7 @@ static const EpdFontData ubuntu_12_regular = {
    false,
    nullptr,
    0,
+    nullptr,
    ubuntu_12_regularKernLeftClasses,
    ubuntu_12_regularKernRightClasses,
    ubuntu_12_regularKernMatrix,
@@ -689,7 +689,38 @@ print(f"ligatures: {len(ligature_pairs)} pairs extracted", file=sys.stderr)

 compress = args.compress

+
+def to_byte_aligned(packed, width, height):
+    """Convert packed 2-bit bitmap to byte-aligned format (rows padded to byte boundary).
+
+    In packed format, pixels flow continuously across row boundaries (4 pixels/byte).
+    In byte-aligned format, each row starts at a byte boundary, padding the last byte
+    of each row with zero bits if width % 4 != 0. This improves DEFLATE compression
+    because identical pixel rows produce identical byte patterns regardless of position.
+    """
+    if width == 0 or height == 0:
+        return b''
+    row_stride = (width + 3) // 4  # bytes per byte-aligned row
+    aligned = bytearray(row_stride * height)
+    for y in range(height):
+        for x in range(width):
+            # Read pixel from packed format (continuous bit stream)
+            packed_pos = y * width + x
+            packed_byte_idx = packed_pos // 4
+            packed_shift = (3 - (packed_pos % 4)) * 2
+            pixel = (packed[packed_byte_idx] >> packed_shift) & 0x3
+
+            # Write pixel to byte-aligned format (row-aligned)
+            aligned_byte_idx = y * row_stride + x // 4
+            aligned_shift = (3 - (x % 4)) * 2
+            aligned[aligned_byte_idx] |= (pixel << aligned_shift)
+    return bytes(aligned)
+
+
 # Build groups for compression
+if compress and not is2Bit:
+    print("Error: --compress requires --2bit (byte-aligned compression only supports 2-bit format)", file=sys.stderr)
+    sys.exit(1)
 if compress:
    # Script-based grouping: glyphs that co-occur in typical text rendering
    # are grouped together for efficient LRU caching on the embedded target.
@@ -747,11 +778,12 @@ if compress:

    for first_idx, count in groups:
        # Concatenate bitmap data for this group
-        group_data = b''
+        packed_len = 0
+        group_aligned = bytearray()
        for gi in range(first_idx, first_idx + count):
            props, packed = all_glyphs[gi]
-            # Update glyph's dataOffset to be within-group offset
-            within_group_offset = len(group_data)
+            # Update glyph's dataOffset to be within-group offset (packed offset)
+            within_group_offset = packed_len
            old_props = modified_glyph_props[gi]
            modified_glyph_props[gi] = GlyphProps(
                width=old_props.width,
@@ -763,13 +795,14 @@ if compress:
                data_offset=within_group_offset,
                code_point=old_props.code_point,
            )
-            group_data += packed
+            packed_len += len(packed)
+            group_aligned.extend(to_byte_aligned(packed, old_props.width, old_props.height))

-        # Compress with raw DEFLATE (no zlib/gzip header)
+        # Compress byte-aligned data with raw DEFLATE (no zlib/gzip header)
        compressor = zlib.compressobj(level=9, wbits=-15)
-        compressed = compressor.compress(group_data) + compressor.flush()
+        compressed = compressor.compress(bytes(group_aligned)) + compressor.flush()

-        compressed_groups.append((compressed, len(group_data), count, first_idx))
+        compressed_groups.append((compressed, len(group_aligned), count, first_idx))
        compressed_bitmap_data.extend(compressed)
        compressed_offset += len(compressed)

@@ -862,8 +895,10 @@ if compress:
    print(f"    {font_name}Groups,")
    print(f"    {len(compressed_groups)},")
 else:
-    print(f"    nullptr,")
-    print(f"    0,")
+    print("    nullptr,")
+    print("    0,")
+# glyphToGroup (not used for script-grouped fonts)
+print("    nullptr,")
 if kern_map:
    print(f"    {font_name}KernLeftClasses,")
    print(f"    {font_name}KernRightClasses,")
@@ -3,9 +3,13 @@
 Round-trip verification for compressed font headers.

 Parses each generated .h file in the given directory, identifies compressed fonts
-(those with a Groups array), decompresses each group, and verifies that
-decompression succeeds and all glyph offsets/lengths fall within bounds.
+(those with a Groups array), decompresses each group (byte-aligned bitmap format),
+compacts to packed format, and verifies the data matches expected glyph sizes.
+
+Supports both contiguous-group fonts (Latin) and frequency-grouped fonts (CJK)
+with glyphToGroup mapping arrays.
 """
+import math
 import os
 import re
 import sys
@@ -18,6 +22,11 @@ def parse_hex_array(text):
    return bytes(int(h, 16) for h in hex_vals)


+def parse_uint8_array(text):
+    """Extract uint8/uint16 values from a C array string like '{ 0, 1, 0xFF, ... }'"""
+    return [int(v, 0) for v in re.findall(r'\b0x[0-9A-Fa-f]+\b|\b\d+\b', text)]
+
+
 def parse_groups(text):
    """Parse EpdFontGroup array entries: { compressedOffset, compressedSize, uncompressedSize, glyphCount, firstGlyphIndex }"""
    groups = []
@@ -48,6 +57,45 @@ def parse_glyphs(text):
    return glyphs


+def get_group_glyph_indices(group, group_index, glyphs, glyph_to_group):
+    """Get the ordered list of glyph indices belonging to a group."""
+    if glyph_to_group is not None:
+        # Frequency-grouped: scan all glyphs
+        return [i for i in range(len(glyphs)) if glyph_to_group[i] == group_index]
+    else:
+        # Contiguous: sequential from firstGlyphIndex
+        first = group['firstGlyphIndex']
+        return list(range(first, first + group['glyphCount']))
+
+
+def compact_aligned_to_packed(aligned_data, width, height):
+    """Convert byte-aligned 2-bit bitmap to packed format (reverse of to_byte_aligned).
+
+    In byte-aligned format, each row starts at a byte boundary.
+    In packed format, pixels flow continuously across row boundaries (4 pixels/byte).
+    """
+    if width == 0 or height == 0:
+        return b''
+    packed_size = math.ceil(width * height / 4)
+    packed = bytearray(packed_size)
+    row_stride = (width + 3) // 4  # bytes per byte-aligned row
+
+    for y in range(height):
+        for x in range(width):
+            # Read pixel from byte-aligned format (row-aligned)
+            aligned_byte_idx = y * row_stride + x // 4
+            aligned_shift = (3 - (x % 4)) * 2
+            pixel = (aligned_data[aligned_byte_idx] >> aligned_shift) & 0x3
+
+            # Write pixel to packed format (continuous bit stream)
+            packed_pos = y * width + x
+            packed_byte_idx = packed_pos // 4
+            packed_shift = (3 - (packed_pos % 4)) * 2
+            packed[packed_byte_idx] |= (pixel << packed_shift)
+
+    return bytes(packed)
+
+
 def verify_font_file(filepath):
    """Verify a single font header file. Returns (font_name, success, message)."""
    with open(filepath, 'r') as f:
@@ -92,6 +140,20 @@ def verify_font_file(filepath):

    glyphs = parse_glyphs(glyphs_match.group(1))

+    # Check for glyphToGroup array (frequency-grouped fonts)
+    glyph_to_group = None
+    g2g_match = re.search(
+        r'static const uint16_t ' + re.escape(font_name) + r'GlyphToGroup\[\]\s*=\s*\{(.+?)\};',
+        content, re.DOTALL
+    )
+    if g2g_match:
+        glyph_to_group = parse_uint8_array(g2g_match.group(1))
+        if len(glyph_to_group) != len(glyphs):
+            return (font_name, False, f"glyphToGroup length ({len(glyph_to_group)}) != glyph count ({len(glyphs)})")
+        max_group_id = max(glyph_to_group)
+        if max_group_id >= len(groups):
+            return (font_name, False, f"glyphToGroup contains group ID {max_group_id} but only {len(groups)} groups exist")
+
    # Verify each group
    for gi, group in enumerate(groups):
        # Extract compressed chunk
@@ -99,7 +161,7 @@ def verify_font_file(filepath):
        if len(chunk) != group['compressedSize']:
            return (font_name, False, f"group {gi}: compressed data truncated (expected {group['compressedSize']}, got {len(chunk)})")

-        # Decompress with raw DEFLATE
+        # Decompress with raw DEFLATE — result is byte-aligned data
        try:
            decompressed = zlib.decompress(chunk, -15)
        except zlib.error as e:
@@ -108,22 +170,64 @@ def verify_font_file(filepath):
        if len(decompressed) != group['uncompressedSize']:
            return (font_name, False, f"group {gi}: size mismatch (expected {group['uncompressedSize']}, got {len(decompressed)})")

-        # Verify each glyph's data within the group
-        first = group['firstGlyphIndex']
-        for j in range(group['glyphCount']):
-            glyph_idx = first + j
+        # Get glyph indices for this group
+        group_glyph_indices = get_group_glyph_indices(group, gi, glyphs, glyph_to_group)
+        if glyph_to_group is not None and len(group_glyph_indices) != group['glyphCount']:
+            return (font_name, False,
+                    f"group {gi}: glyphCount {group['glyphCount']} != mapping count {len(group_glyph_indices)}")
+
+        # Walk through byte-aligned data, compact each glyph, and verify against packed format
+        byte_aligned_offset = 0
+        packed_offset = 0
+
+        for glyph_idx in group_glyph_indices:
            if glyph_idx >= len(glyphs):
                return (font_name, False, f"group {gi}: glyph index {glyph_idx} out of range")
-
            glyph = glyphs[glyph_idx]
-            offset = glyph['dataOffset']
-            length = glyph['dataLength']
+            width = glyph['width']
+            height = glyph['height']

-            if offset + length > len(decompressed):
-                return (font_name, False, f"group {gi}, glyph {glyph_idx}: data extends beyond decompressed buffer "
-                        f"(offset={offset}, length={length}, decompressed_size={len(decompressed)})")
+            if width == 0 or height == 0:
+                # Zero-size glyphs should have dataOffset == current packed_offset and dataLength == 0
+                if glyph['dataOffset'] != packed_offset:
+                    return (font_name, False, f"group {gi}, glyph {glyph_idx}: zero-size glyph dataOffset {glyph['dataOffset']} != expected packed offset {packed_offset}")
+                if glyph['dataLength'] != 0:
+                    return (font_name, False, f"group {gi}, glyph {glyph_idx}: zero-size glyph dataLength {glyph['dataLength']} != expected 0")
+                continue

-    return (font_name, True, f"{len(groups)} groups, {len(glyphs)} glyphs OK")
+            aligned_size = ((width + 3) // 4) * height
+            packed_size = math.ceil(width * height / 4)
+
+            # Verify packed offset and size match glyph metadata
+            if glyph['dataOffset'] != packed_offset:
+                return (font_name, False, f"group {gi}, glyph {glyph_idx}: dataOffset {glyph['dataOffset']} != expected packed offset {packed_offset}")
+            if glyph['dataLength'] != packed_size:
+                return (font_name, False, f"group {gi}, glyph {glyph_idx}: dataLength {glyph['dataLength']} != expected packed length {packed_size} "
+                        f"(width={width}, height={height})")
+
+            # Extract byte-aligned data for this glyph
+            if byte_aligned_offset + aligned_size > len(decompressed):
+                return (font_name, False, f"group {gi}, glyph {glyph_idx}: byte-aligned data extends beyond decompressed buffer "
+                        f"(offset={byte_aligned_offset}, size={aligned_size}, buf_size={len(decompressed)})")
+
+            aligned_glyph = decompressed[byte_aligned_offset:byte_aligned_offset + aligned_size]
+
+            # Compact to packed and verify pixel values are valid (0-3 for 2-bit)
+            packed_glyph = compact_aligned_to_packed(aligned_glyph, width, height)
+            if len(packed_glyph) != packed_size:
+                return (font_name, False, f"group {gi}, glyph {glyph_idx}: compacted size {len(packed_glyph)} != expected {packed_size}")
+
+            byte_aligned_offset += aligned_size
+            packed_offset += packed_size
+
+        # Verify total byte-aligned size matches uncompressedSize
+        if byte_aligned_offset != group['uncompressedSize']:
+            return (font_name, False, f"group {gi}: total byte-aligned size {byte_aligned_offset} != uncompressedSize {group['uncompressedSize']}")
+
+    extra_info = ""
+    if glyph_to_group is not None:
+        extra_info = " (frequency-grouped)"
+    return (font_name, True, f"{len(groups)} groups, {len(glyphs)} glyphs OK{extra_info}")


 def main():