fix: Add framebuffer release/realloc and improved lazy indexing (#2563)

bin/clang-format-fix

@@ -47,6 +47,7 @@ git ls-files  --exclude-standard ${GIT_LS_FILES_FLAGS} \
     | grep -v -E '^lib/EpdFont/builtinFonts/' \
     | grep -v -E '^lib/Epub/Epub/hyphenation/generated/' \
     | grep -v -E '^lib/uzlib/' \
+    | grep -v -E '^lib/miniz/third_party/' \
     | xargs -r "${CLANG_FORMAT_BIN}" -style=file -i
 # Restore strict pipeline failure handling for the rest of the script.
 set -o pipefail

lib/Epub/Epub/BookMetadataCache.cpp

@@ -1,5 +1,6 @@
 #include "BookMetadataCache.h"
 
+#include <BufferedFile.h>
 #include <Logging.h>
 #include <Serialization.h>
 #include <Utf8.h>
@@ -14,6 +15,52 @@ constexpr uint8_t BOOK_CACHE_VERSION = 8;  // v8: TOC/book titles stored NFC-com
 constexpr char bookBinFile[] = "/book.bin";
 constexpr char tmpSpineBinFile[] = "/spine.bin.tmp";
 constexpr char tmpTocBinFile[] = "/toc.bin.tmp";
+// Buffer size for the buildBookBin streams. 3 buffers x 4KB, transient (freed on
+// return); 4KB = 8 SD sectors per transfer, enough to stop the sector-cache thrash.
+constexpr size_t BUILD_IO_BUFFER_SIZE = 4096;
+
+// Entry (de)serializers, templated so they run over HalFile and the Buffered*
+// wrappers alike (two instantiations each -- a few hundred bytes of flash, in
+// exchange for the build path streaming at SD speed instead of per-pod).
+template <typename F>
+uint32_t writeSpineEntryTo(F& file, const BookMetadataCache::SpineEntry& entry) {
+  const uint32_t pos = file.position();
+  serialization::writeString(file, entry.href);
+  serialization::writePod(file, entry.cumulativeSize);
+  serialization::writePod(file, entry.tocIndex);
+  return pos;
+}
+
+template <typename F>
+uint32_t writeTocEntryTo(F& file, const BookMetadataCache::TocEntry& entry) {
+  const uint32_t pos = file.position();
+  serialization::writeString(file, entry.title);
+  serialization::writeString(file, entry.href);
+  serialization::writeString(file, entry.anchor);
+  serialization::writePod(file, entry.level);
+  serialization::writePod(file, entry.spineIndex);
+  return pos;
+}
+
+template <typename F>
+BookMetadataCache::SpineEntry readSpineEntryFrom(F& file) {
+  BookMetadataCache::SpineEntry entry;
+  serialization::readString(file, entry.href);
+  serialization::readPod(file, entry.cumulativeSize);
+  serialization::readPod(file, entry.tocIndex);
+  return entry;
+}
+
+template <typename F>
+BookMetadataCache::TocEntry readTocEntryFrom(F& file) {
+  BookMetadataCache::TocEntry entry;
+  serialization::readString(file, entry.title);
+  serialization::readString(file, entry.href);
+  serialization::readString(file, entry.anchor);
+  serialization::readPod(file, entry.level);
+  serialization::readPod(file, entry.spineIndex);
+  return entry;
+}
 }  // namespace
 
 /* ============= WRITING / BUILDING FUNCTIONS ================ */
@@ -30,13 +77,23 @@ bool BookMetadataCache::beginContentOpfPass() {
   LOG_DBG("BMC", "Beginning content opf pass");
 
   // Open spine file for writing
-  return Storage.openFileForWrite("BMC", cachePath + tmpSpineBinFile, spineFile);
+  if (!Storage.openFileForWrite("BMC", cachePath + tmpSpineBinFile, spineFile)) {
+    return false;
+  }
+  // Wrapper OOM is fine: createSpineEntry falls back to unbuffered writes.
+  passOut = makeUniqueNoThrow<serialization::BufferedFileWriter>(spineFile, BUILD_IO_BUFFER_SIZE);
+  return true;
 }
 
 bool BookMetadataCache::endContentOpfPass() {
+  const bool flushed = !passOut || passOut->flush();
+  passOut.reset();
   // Explicit close() required: member variable persists beyond function scope
   spineFile.close();
-  return true;
+  if (!flushed) {
+    LOG_ERR("BMC", "Failed writing spine tmp file");
+  }
+  return flushed;
 }
 
 bool BookMetadataCache::beginTocPass() {
@@ -74,10 +131,17 @@ bool BookMetadataCache::beginTocPass() {
     useSpineHrefIndex = false;
   }
 
+  // Wrapper OOM is fine: createTocEntry falls back to unbuffered writes.
+  passOut = makeUniqueNoThrow<serialization::BufferedFileWriter>(tocFile, BUILD_IO_BUFFER_SIZE);
   return true;
 }
 
 bool BookMetadataCache::endTocPass() {
+  const bool flushed = !passOut || passOut->flush();
+  passOut.reset();
+  if (!flushed) {
+    LOG_ERR("BMC", "Failed writing toc tmp file");
+  }
   // Explicit close() required: member variables persist beyond function scope
   tocFile.close();
   spineFile.close();
@@ -86,7 +150,7 @@ bool BookMetadataCache::endTocPass() {
   spineHrefIndex.shrink_to_fit();
   useSpineHrefIndex = false;
 
-  return true;
+  return flushed;
 }
 
 bool BookMetadataCache::endWrite() {
@@ -119,6 +183,14 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
     return false;
   }
 
+  // Buffered streams for the whole build: every access below is sequential per
+  // file, but interleaved ACROSS files, which thrashes SdFat's single shared
+  // sector cache when unbuffered (one 512B SD transaction per 4-byte pod --
+  // measured 31s for a 1,732-spine omnibus). Three 4KB buffers, freed on return.
+  serialization::BufferedFileWriter bookOut(bookFile, BUILD_IO_BUFFER_SIZE);
+  serialization::BufferedFileReader spineIn(spineFile, BUILD_IO_BUFFER_SIZE);
+  serialization::BufferedFileReader tocIn(tocFile, BUILD_IO_BUFFER_SIZE);
+
   constexpr uint32_t headerASize =
       sizeof(BOOK_CACHE_VERSION) + /* LUT Offset */ sizeof(uint32_t) + sizeof(spineCount) + sizeof(tocCount);
   const uint32_t metadataSize = metadata.title.size() + metadata.author.size() + metadata.language.size() +
@@ -128,31 +200,34 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
   const uint32_t lutOffset = headerASize + metadataSize;
 
   // Header A
-  serialization::writePod(bookFile, BOOK_CACHE_VERSION);
+  serialization::writePod(bookOut, BOOK_CACHE_VERSION);
-  serialization::writePod(bookFile, lutOffset);
+  serialization::writePod(bookOut, lutOffset);
-  serialization::writePod(bookFile, spineCount);
+  serialization::writePod(bookOut, spineCount);
-  serialization::writePod(bookFile, tocCount);
+  serialization::writePod(bookOut, tocCount);
   // Metadata
-  serialization::writeString(bookFile, metadata.title);
+  serialization::writeString(bookOut, metadata.title);
-  serialization::writeString(bookFile, metadata.author);
+  serialization::writeString(bookOut, metadata.author);
-  serialization::writeString(bookFile, metadata.language);
+  serialization::writeString(bookOut, metadata.language);
-  serialization::writeString(bookFile, metadata.coverItemHref);
+  serialization::writeString(bookOut, metadata.coverItemHref);
-  serialization::writeString(bookFile, metadata.textReferenceHref);
+  serialization::writeString(bookOut, metadata.textReferenceHref);
 
   // Loop through spine entries, writing LUT positions
-  spineFile.seek(0);
+  spineIn.seek(0);
   for (int i = 0; i < spineCount; i++) {
-    uint32_t pos = spineFile.position();
+    const uint32_t pos = spineIn.position();
-    auto spineEntry = readSpineEntry(spineFile);
+    readSpineEntryFrom(spineIn);
-    serialization::writePod(bookFile, pos + lutOffset + lutSize);
+    serialization::writePod(bookOut, pos + lutOffset + lutSize);
   }
+  // Total size of the spine tmp file: entries land in book.bin after the toc LUT
+  // and the full spine block, so toc LUT positions are offset by it.
+  const auto spineBytes = static_cast<uint32_t>(spineIn.position());
 
   // Loop through toc entries, writing LUT positions
-  tocFile.seek(0);
+  tocIn.seek(0);
   for (int i = 0; i < tocCount; i++) {
-    uint32_t pos = tocFile.position();
+    const uint32_t pos = tocIn.position();
-    auto tocEntry = readTocEntry(tocFile);
+    readTocEntryFrom(tocIn);
-    serialization::writePod(bookFile, pos + lutOffset + lutSize + static_cast<uint32_t>(spineFile.position()));
+    serialization::writePod(bookOut, pos + lutOffset + lutSize + spineBytes);
   }
 
   // LUTs complete
@@ -160,9 +235,9 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
 
   // Build spineIndex->tocIndex mapping in one pass (O(n) instead of O(n*m))
   std::deque<int16_t> spineToTocIndex(spineCount, -1);
-  tocFile.seek(0);
+  tocIn.seek(0);
   for (int j = 0; j < tocCount; j++) {
-    auto tocEntry = readTocEntry(tocFile);
+    auto tocEntry = readTocEntryFrom(tocIn);
     if (tocEntry.spineIndex >= 0 && tocEntry.spineIndex < spineCount) {
       if (spineToTocIndex[tocEntry.spineIndex] == -1) {
         spineToTocIndex[tocEntry.spineIndex] = static_cast<int16_t>(j);
@@ -197,9 +272,9 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
     std::deque<ZipFile::SizeTarget> targets;
     targets.resize(spineCount);
 
-    spineFile.seek(0);
+    spineIn.seek(0);
     for (int i = 0; i < spineCount; i++) {
-      auto entry = readSpineEntry(spineFile);
+      auto entry = readSpineEntryFrom(spineIn);
       std::string path = FsHelpers::normalisePath(entry.href);
 
       ZipFile::SizeTarget t;
@@ -224,10 +299,10 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
   }
 
   uint32_t cumSize = 0;
-  spineFile.seek(0);
+  spineIn.seek(0);
   int lastSpineTocIndex = -1;
   for (int i = 0; i < spineCount; i++) {
-    auto spineEntry = readSpineEntry(spineFile);
+    auto spineEntry = readSpineEntryFrom(spineIn);
 
     spineEntry.tocIndex = spineToTocIndex[i];
 
@@ -260,23 +335,33 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
     spineEntry.cumulativeSize = cumSize;
 
     // Write out spine data to book.bin
-    writeSpineEntry(bookFile, spineEntry);
+    writeSpineEntryTo(bookOut, spineEntry);
   }
   // Close opened zip file
   zip.close();
 
   // Loop through toc entries from toc file writing to book.bin
-  tocFile.seek(0);
+  tocIn.seek(0);
   for (int i = 0; i < tocCount; i++) {
-    auto tocEntry = readTocEntry(tocFile);
+    auto tocEntry = readTocEntryFrom(tocIn);
-    writeTocEntry(bookFile, tocEntry);
+    writeTocEntryTo(bookOut, tocEntry);
   }
 
+  const bool written = bookOut.flush();
+
   // Explicit close() required: member variables persist beyond function scope
   bookFile.close();
   spineFile.close();
   tocFile.close();
 
+  if (!written) {
+    // A short write (card full/removed) would leave a truncated book.bin that
+    // still passes the version check on load; remove it so the next open rebuilds.
+    LOG_ERR("BMC", "Failed writing book.bin, removing truncated file");
+    Storage.remove((cachePath + bookBinFile).c_str());
+    return false;
+  }
+
   LOG_DBG("BMC", "Successfully built book.bin");
   return true;
 }
@@ -294,21 +379,11 @@ bool BookMetadataCache::cleanupTmpFiles() const {
 }
 
 uint32_t BookMetadataCache::writeSpineEntry(HalFile& file, const SpineEntry& entry) const {
-  const uint32_t pos = file.position();
+  return writeSpineEntryTo(file, entry);
-  serialization::writeString(file, entry.href);
-  serialization::writePod(file, entry.cumulativeSize);
-  serialization::writePod(file, entry.tocIndex);
-  return pos;
 }
 
 uint32_t BookMetadataCache::writeTocEntry(HalFile& file, const TocEntry& entry) const {
-  const uint32_t pos = file.position();
+  return writeTocEntryTo(file, entry);
-  serialization::writeString(file, entry.title);
-  serialization::writeString(file, entry.href);
-  serialization::writeString(file, entry.anchor);
-  serialization::writePod(file, entry.level);
-  serialization::writePod(file, entry.spineIndex);
-  return pos;
 }
 
 // Note: for the LUT to be accurate, this **MUST** be called for all spine items before `addTocEntry` is ever called
@@ -320,7 +395,11 @@ void BookMetadataCache::createSpineEntry(const std::string& href) {
   }
 
   const SpineEntry entry(href, 0, -1);
-  writeSpineEntry(spineFile, entry);
+  if (passOut) {
+    writeSpineEntryTo(*passOut, entry);
+  } else {
+    writeSpineEntry(spineFile, entry);
+  }
   spineCount++;
 }
 
@@ -368,7 +447,11 @@ void BookMetadataCache::createTocEntry(const std::string& title, const std::stri
   // Compose the title to NFC at index time so the cache stores precomposed glyphs;
   // device fonts have no combining-mark positioning, so NFD titles render broken.
   const TocEntry entry(utf8ComposeNfc(title), href, anchor, level, spineIndex);
-  writeTocEntry(tocFile, entry);
+  if (passOut) {
+    writeTocEntryTo(*passOut, entry);
+  } else {
+    writeTocEntry(tocFile, entry);
+  }
   tocCount++;
 }
 
@@ -442,19 +525,7 @@ BookMetadataCache::TocEntry BookMetadataCache::getTocEntry(const int index) {
 }
 
 BookMetadataCache::SpineEntry BookMetadataCache::readSpineEntry(HalFile& file) const {
-  SpineEntry entry;
+  return readSpineEntryFrom(file);
-  serialization::readString(file, entry.href);
-  serialization::readPod(file, entry.cumulativeSize);
-  serialization::readPod(file, entry.tocIndex);
-  return entry;
 }
 
-BookMetadataCache::TocEntry BookMetadataCache::readTocEntry(HalFile& file) const {
+BookMetadataCache::TocEntry BookMetadataCache::readTocEntry(HalFile& file) const { return readTocEntryFrom(file); }
-  TocEntry entry;
-  serialization::readString(file, entry.title);
-  serialization::readString(file, entry.href);
-  serialization::readString(file, entry.anchor);
-  serialization::readPod(file, entry.level);
-  serialization::readPod(file, entry.spineIndex);
-  return entry;
-}

lib/Epub/Epub/BookMetadataCache.h

@@ -1,9 +1,11 @@
 #pragma once
 
+#include <BufferedFile.h>
 #include <HalStorage.h>
 
 #include <algorithm>
 #include <deque>
+#include <memory>
 #include <string>
 
 class BookMetadataCache {
@@ -54,6 +56,11 @@ class BookMetadataCache {
   // Temp file handles during build
   HalFile spineFile;
   HalFile tocFile;
+  // Buffers the per-entry tmp-file writes during the OPF/TOC passes: those
+  // writes interleave with zip-inflate SD reads, and unbuffered they thrash
+  // SdFat's shared sector cache (one 512B transaction per 4-byte pod). One
+  // wrapper serves whichever pass is active (spine, then toc).
+  std::unique_ptr<serialization::BufferedFileWriter> passOut;
 
   // Index for fast href→spineIndex lookup (used only for large EPUBs)
   struct SpineHrefIndexEntry {

lib/GfxRenderer/GfxRenderer.cpp

@@ -1,6 +1,7 @@
 #include "GfxRenderer.h"
 
 #include <BidiUtils.h>
+#include <BuildScratch.h>
 #include <FontDecompressor.h>
 #include <HalGPIO.h>
 #include <Logging.h>
@@ -91,6 +92,47 @@ void GfxRenderer::begin() {
   bwBufferChunks.assign((frameBufferSize + BW_BUFFER_CHUNK_SIZE - 1) / BW_BUFFER_CHUNK_SIZE, nullptr);
 }
 
+void GfxRenderer::releaseFrameBufferForBuild() {
+  // Lend the framebuffer's bytes IN PLACE: the allocation is never freed, so
+  // it cannot move and repeated loans cannot fragment the heap (the previous
+  // free+realloc model measurably decayed the max contiguous block over a
+  // session). The bytes are deposited in the build-scratch registry so
+  // memory-hungry build phases (e.g. InflateStream's tinfl state + window)
+  // can claim them instead of allocating.
+  uint32_t size = 0;
+  uint8_t* scratch = display.lendFrameBufferStorage(&size);
+  frameBuffer = nullptr;
+  if (scratch) {
+    buildscratch::lend(scratch, size);
+  }
+}
+
+bool GfxRenderer::restoreFrameBufferAfterBuild() {
+  buildscratch::reclaim();
+  display.returnFrameBufferStorage();  // cannot fail: the allocation was never freed
+  frameBuffer = display.getFrameBuffer();
+  return frameBuffer != nullptr;
+}
+
+GfxRenderer::FrameBufferLoan::FrameBufferLoan(GfxRenderer& renderer) : renderer_(renderer) {
+  // Nesting guard: if the framebuffer is already lent out (an outer loan),
+  // stay inert so this end() cannot return storage the outer loan still owns.
+  if (!renderer_.hasFrameBuffer()) return;
+  renderer_.releaseFrameBufferForBuild();
+  active_ = true;
+}
+
+void GfxRenderer::FrameBufferLoan::end() {
+  if (!active_) return;
+  active_ = false;
+  if (!renderer_.restoreFrameBufferAfterBuild()) {
+    // Only reachable if the framebuffer never existed, which begin() already
+    // asserts against; kept as a backstop since running blind helps nobody.
+    LOG_ERR("GFX", "Framebuffer restore failed - restarting");
+    ESP.restart();
+  }
+}
+
 bool GfxRenderer::isFontCacheScanning() const { return fontCacheManager_ && fontCacheManager_->isScanning(); }
 
 void GfxRenderer::insertFont(const int fontId, EpdFontFamily font) {

lib/GfxRenderer/GfxRenderer.h

@@ -250,6 +250,34 @@ class GfxRenderer {
   // Font helpers
   const uint8_t* getGlyphBitmap(const EpdFontData* fontData, const EpdGlyph* glyph) const;
 
+  // Lend the 48 KB framebuffer's bytes to a memory-hungry phase (chapter
+  // builds) WITHOUT freeing the allocation, so it never moves and repeated
+  // loans cannot fragment the heap. Between release and restore NOTHING may
+  // draw or display — the panel keeps showing its last refreshed image. The
+  // lent bytes are published via buildscratch::claim() for consumers like
+  // InflateStream. restore returns the buffer white, so the caller must
+  // redraw the full screen; it cannot fail (no allocation involved).
+  void releaseFrameBufferForBuild();
+  bool restoreFrameBufferAfterBuild();
+  bool hasFrameBuffer() const { return frameBuffer != nullptr; }
+
+  // RAII form of the loan above, for blocking build regions with early-return
+  // error paths: restores on scope exit (or explicitly via end()). Display the
+  // popup/screen the panel should hold BEFORE constructing one. Constructing
+  // while the framebuffer is already lent yields an inert loan (nesting-safe).
+  class FrameBufferLoan {
+   public:
+    explicit FrameBufferLoan(GfxRenderer& renderer);
+    ~FrameBufferLoan() { end(); }
+    void end();
+    FrameBufferLoan(const FrameBufferLoan&) = delete;
+    FrameBufferLoan& operator=(const FrameBufferLoan&) = delete;
+
+   private:
+    GfxRenderer& renderer_;
+    bool active_ = false;
+  };
+
   // Low level functions
   uint8_t* getFrameBuffer() const;
   size_t getBufferSize() const;

lib/InflateReader/InflateReader.h

@@ -13,6 +13,11 @@ enum class InflateStatus {
 
 // Streaming deflate decompressor wrapping uzlib.
 //
+// NOTE: retained ONLY for FontDecompressor's tiny one-shot flash-resident group
+// decompressions, where uzlib's ~1KB state beats tinfl's ~11KB on the
+// OOM-sensitive render path. All throughput paths (zip entries, PNG IDAT) use
+// InflateStream (lib/miniz), which decodes several times faster.
+//
 // Two modes:
 //   init(false)  — one-shot: input is a contiguous buffer, call read() once.
 //   init(true)   — streaming: allocates a 32KB ring buffer for back-references

lib/Memory/BuildScratch.cpp

@@ -0,0 +1,51 @@
+#include "BuildScratch.h"
+
+#include <Logging.h>
+
+#include <atomic>
+
+namespace buildscratch {
+namespace {
+uint8_t* block = nullptr;
+size_t blockLen = 0;
+// atomic exchange so an opportunistic claim from another task can never
+// double-hand-out the block (single core, but FreeRTOS preempts).
+std::atomic<bool> claimed{false};
+}  // namespace
+
+void lend(uint8_t* buf, const size_t len) {
+  if (block) {
+    LOG_ERR("SCR", "Build scratch lent twice; ignoring second lend");
+    return;
+  }
+  block = buf;
+  blockLen = len;
+  claimed.store(false);
+}
+
+void reclaim() {
+  if (claimed.load()) {
+    // A consumer still holds the block. The storage stays valid (it is the
+    // framebuffer allocation, never freed) but its contents are about to be
+    // clobbered; the consumer's output will be garbage. Loud log so a
+    // lifetime bug is visible instead of a silent corrupt decode.
+    LOG_ERR("SCR", "Build scratch reclaimed while still claimed");
+  }
+  block = nullptr;
+  blockLen = 0;
+  claimed.store(false);
+}
+
+uint8_t* claim(const size_t minLen, size_t* lenOut) {
+  if (!block || blockLen < minLen) return nullptr;
+  bool expected = false;
+  if (!claimed.compare_exchange_strong(expected, true)) return nullptr;
+  if (lenOut) *lenOut = blockLen;
+  return block;
+}
+
+void release(const uint8_t* p) {
+  if (p && p == block) claimed.store(false);
+}
+
+}  // namespace buildscratch

lib/Memory/BuildScratch.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// Registry for the framebuffer bytes lent out during a build phase
+// (GfxRenderer::FrameBufferLoan). The lender (GfxRenderer) deposits the block
+// with lend()/reclaim(); a memory-hungry consumer (e.g. InflateStream's ~43KB
+// tinfl state + window) may claim() it instead of allocating from the heap.
+//
+// Exactly one claimant at a time; claim() returns nullptr when the block is
+// absent or already claimed, and consumers must fall back to the heap. The
+// underlying storage is the framebuffer allocation itself, which is never
+// freed -- so even the pathological case (reclaim() while still claimed, which
+// logs an error) reads garbage, never freed memory.
+namespace buildscratch {
+
+// Lender side (GfxRenderer only).
+void lend(uint8_t* buf, size_t len);
+void reclaim();
+
+// Consumer side: exclusive claim of the whole block if it is at least minLen
+// bytes; nullptr means "use the heap". Release with the same pointer.
+uint8_t* claim(size_t minLen, size_t* lenOut = nullptr);
+void release(const uint8_t* p);
+
+}  // namespace buildscratch

lib/PngToBmpConverter/PngToBmpConverter.cpp

@@ -2,7 +2,7 @@
 
 #include <HalDisplay.h>
 #include <HalStorage.h>
-#include <InflateReader.h>
+#include <InflateStream.h>
 #include <Logging.h>
 
 #include <cstdio>
@@ -174,9 +174,8 @@ void writeBmpHeader2bit(Print& bmpOut, const int width, const int height) {
 }  // namespace
 
 // Context for streaming PNG decompression
-// IMPORTANT: reader must be the first field - the uzlib callback casts uzlib_uncomp* to PngDecodeContext*
 struct PngDecodeContext {
-  InflateReader reader;  // Must be first — callback casts uzlib_uncomp* to PngDecodeContext*
+  InflateStream reader;
   HalFile* file;
 
   // PNG image properties
@@ -195,7 +194,7 @@ struct PngDecodeContext {
   uint32_t chunkBytesRemaining;  // bytes left in current IDAT chunk
   bool idatFinished;             // no more IDAT chunks
 
-  // File read buffer for feeding uzlib
+  // File read buffer for feeding the inflate stream
   uint8_t readBuf[2048];
 
   // Palette for indexed color (type 3)
@@ -229,21 +228,21 @@ static bool findNextIdatChunk(PngDecodeContext& ctx) {
   }
 }
 
-// uzlib callback: reads the next batch of IDAT data from the file
+// Fill callback: reads the next batch of IDAT data from the file
-static int pngIdatReadCallback(uzlib_uncomp* uncomp) {
+static size_t pngIdatFillCallback(void* vctx, const uint8_t** data) {
-  auto* ctx = reinterpret_cast<PngDecodeContext*>(uncomp);
+  auto* ctx = static_cast<PngDecodeContext*>(vctx);
 
-  if (ctx->idatFinished) return -1;
+  if (ctx->idatFinished) return 0;
 
   // Skip 4-byte CRC and find next IDAT chunk when current chunk is exhausted
   while (ctx->chunkBytesRemaining == 0) {
     if (!ctx->file->seekCur(4)) {  // skip 4-byte CRC of previous IDAT
       ctx->idatFinished = true;
-      return -1;
+      return 0;
     }
     if (!findNextIdatChunk(*ctx)) {
       ctx->idatFinished = true;
-      return -1;
+      return 0;
     }
   }
 
@@ -251,18 +250,15 @@ static int pngIdatReadCallback(uzlib_uncomp* uncomp) {
   size_t toRead = sizeof(ctx->readBuf);
   if (toRead > ctx->chunkBytesRemaining) toRead = ctx->chunkBytesRemaining;
 
-  int bytesRead = ctx->file->read(ctx->readBuf, toRead);
+  const int bytesRead = ctx->file->read(ctx->readBuf, toRead);
   if (bytesRead <= 0) {
     ctx->idatFinished = true;
-    return -1;
+    return 0;
   }
 
   ctx->chunkBytesRemaining -= bytesRead;
-
+  *data = ctx->readBuf;
-  // Give uzlib the buffer (skip first byte since we return it directly)
+  return static_cast<size_t>(bytesRead);
-  uncomp->source = ctx->readBuf + 1;
-  uncomp->source_limit = ctx->readBuf + bytesRead;
-  return ctx->readBuf[0];
 }
 
 // Decode one scanline: decompress filter byte + raw bytes, then unfilter
@@ -555,16 +551,16 @@ bool PngToBmpConverter::pngFileToBmpStreamInternal(HalFile& pngFile, Print& bmpO
     return false;
   }
 
-  // Initialize streaming decompressor with 32KB ring buffer for back-reference history
+  // Initialize streaming decompressor with 32KB window for back-reference history
   if (!ctx.reader.init(true)) {
-    LOG_ERR("PNG", "Failed to init inflate reader");
+    LOG_ERR("PNG", "Failed to init inflate stream");
     free(ctx.currentRow);
     free(ctx.previousRow);
     return false;
   }
-  ctx.reader.setReadCallback(pngIdatReadCallback);
+  ctx.reader.setFill(pngIdatFillCallback, &ctx);
-  // PNG IDAT data is zlib-wrapped: consume the 2-byte zlib header (CMF + FLG)
+  // PNG IDAT data is zlib-wrapped (2-byte header + trailing adler32)
-  ctx.reader.skipZlibHeader();
+  ctx.reader.setZlibWrapped();
 
   // Calculate output dimensions (same logic as JpegToBmpConverter)
   int outWidth = width;

lib/Serialization/BufferedFile.h

@@ -0,0 +1,161 @@
+#pragma once
+#include <HalStorage.h>
+#include <Memory.h>
+
+#include <algorithm>
+#include <cstring>
+#include <string>
+
+namespace serialization {
+
+// Sequential buffered wrappers over HalFile.
+//
+// SdFat keeps ONE shared 512-byte sector cache per volume, so interleaving small
+// reads/writes across two or more files evicts and reloads that sector on nearly
+// every call -- each 4-byte pod becomes a full SD transaction (measured: 31s to
+// stream ~200KB through BookMetadataCache::buildBookBin on a 1,732-spine EPUB).
+// Batching into chunk-sized transfers keeps each file at sequential SD speed.
+//
+// Heap: one fixed buffer per wrapper, allocated once at construction and freed at
+// scope exit. If the allocation fails the wrapper degrades to unbuffered
+// passthrough -- correct, just slow -- so callers never need an OOM path.
+//
+// Constraint: the wrapper must be the file's ONLY accessor while alive (it tracks
+// the underlying position itself); mixing direct HalFile calls in desynchronizes it.
+
+class BufferedFileWriter {
+ public:
+  BufferedFileWriter(HalFile& file, const size_t capacity)
+      : file(file), buf(makeUniqueNoThrow<uint8_t[]>(capacity)), cap(buf ? capacity : 0), pos(file.position()) {}
+  ~BufferedFileWriter() { flush(); }
+  BufferedFileWriter(const BufferedFileWriter&) = delete;
+  BufferedFileWriter& operator=(const BufferedFileWriter&) = delete;
+
+  void write(const void* src, const size_t len) {
+    pos += len;
+    const auto* p = static_cast<const uint8_t*>(src);
+    if (fill + len > cap) {
+      flushBuffer();
+    }
+    if (len >= cap) {  // also the cap == 0 passthrough
+      okFlag &= file.write(p, len) == len;
+      return;
+    }
+    // Typed local: cppcheck misreads unique_ptr<uint8_t[]>::get() arithmetic as void*.
+    uint8_t* const data = buf.get();
+    memcpy(data + fill, p, len);
+    fill += len;
+  }
+
+  // Logical write position (bytes written since the file was opened).
+  size_t position() const { return pos; }
+
+  // Flush buffered bytes; returns false if any write so far has failed short.
+  bool flush() {
+    flushBuffer();
+    return okFlag;
+  }
+
+ private:
+  void flushBuffer() {
+    if (fill == 0) return;
+    okFlag &= file.write(buf.get(), fill) == fill;
+    fill = 0;
+  }
+
+  HalFile& file;
+  std::unique_ptr<uint8_t[]> buf;
+  const size_t cap;
+  size_t fill = 0;
+  size_t pos;
+  bool okFlag = true;
+};
+
+class BufferedFileReader {
+ public:
+  BufferedFileReader(HalFile& file, const size_t capacity)
+      : file(file), buf(makeUniqueNoThrow<uint8_t[]>(capacity)), cap(buf ? capacity : 0), bufStart(file.position()) {}
+  BufferedFileReader(const BufferedFileReader&) = delete;
+  BufferedFileReader& operator=(const BufferedFileReader&) = delete;
+
+  size_t read(void* dst, size_t len) {
+    auto* p = static_cast<uint8_t*>(dst);
+    if (cap == 0) {  // passthrough
+      const int n = file.read(p, len);
+      const size_t got = n < 0 ? 0 : static_cast<size_t>(n);
+      bufStart += got;
+      return got;
+    }
+    size_t total = 0;
+    while (len > 0) {
+      if (off == fill) {
+        bufStart += fill;
+        off = 0;
+        const int n = file.read(buf.get(), cap);
+        fill = n < 0 ? 0 : static_cast<size_t>(n);
+        if (fill == 0) break;  // EOF or error
+      }
+      const size_t chunk = std::min(len, fill - off);
+      // Typed local: cppcheck misreads unique_ptr<uint8_t[]>::get() arithmetic as void*.
+      const uint8_t* const data = buf.get();
+      memcpy(p, data + off, chunk);
+      p += chunk;
+      off += chunk;
+      len -= chunk;
+      total += chunk;
+    }
+    return total;
+  }
+
+  // Logical read position.
+  size_t position() const { return bufStart + off; }
+
+  bool seek(const size_t target) {
+    // Within the buffered window: just move the cursor.
+    if (cap != 0 && target >= bufStart && target < bufStart + fill) {
+      off = target - bufStart;
+      return true;
+    }
+    if (!file.seek(target)) return false;
+    bufStart = target;
+    fill = 0;
+    off = 0;
+    return true;
+  }
+
+ private:
+  HalFile& file;
+  std::unique_ptr<uint8_t[]> buf;
+  const size_t cap;
+  size_t fill = 0;
+  size_t off = 0;
+  size_t bufStart;
+};
+
+// serialization:: overloads mirroring the HalFile ones in Serialization.h.
+template <typename T>
+void writePod(BufferedFileWriter& out, const T& value) {
+  out.write(&value, sizeof(T));
+}
+
+template <typename T>
+void readPod(BufferedFileReader& in, T& value) {
+  in.read(&value, sizeof(T));
+}
+
+inline void writeString(BufferedFileWriter& out, const std::string& s) {
+  const uint32_t len = s.size();
+  writePod(out, len);
+  out.write(s.data(), len);
+}
+
+inline void readString(BufferedFileReader& in, std::string& s) {
+  uint32_t len;
+  readPod(in, len);
+  s.resize(len);
+  if (len > 0) {
+    in.read(&s[0], len);
+  }
+}
+
+}  // namespace serialization

lib/ZipFile/ZipFile.cpp

@@ -1,13 +1,12 @@
 #include "ZipFile.h"
 
 #include <HalStorage.h>
-#include <InflateReader.h>
+#include <InflateStream.h>
 #include <Logging.h>
 
 #include <algorithm>
 
 struct ZipInflateCtx {
-  InflateReader reader;  // Must be first — callback casts uzlib_uncomp* to ZipInflateCtx*
   HalFile* file = nullptr;
   size_t fileRemaining = 0;
   uint8_t* readBuf = nullptr;
@@ -40,19 +39,16 @@ class ScopedOpenClose final {
   bool ok = true;  // true when zip was already open (no open() call needed)
 };
 
-int zipReadCallback(uzlib_uncomp* uncomp) {
+size_t zipFillCallback(void* vctx, const uint8_t** data) {
-  auto* ctx = reinterpret_cast<ZipInflateCtx*>(uncomp);
+  auto* ctx = static_cast<ZipInflateCtx*>(vctx);
-  if (ctx->fileRemaining == 0) return -1;
+  if (ctx->fileRemaining == 0) return 0;
 
   const size_t toRead = ctx->fileRemaining < ctx->readBufSize ? ctx->fileRemaining : ctx->readBufSize;
   const size_t bytesRead = ctx->file->read(ctx->readBuf, toRead);
   ctx->fileRemaining -= bytesRead;
 
-  if (bytesRead == 0) return -1;
+  *data = ctx->readBuf;
-
+  return bytesRead;
-  uncomp->source = ctx->readBuf + 1;
-  uncomp->source_limit = ctx->readBuf + bytesRead;
-  return ctx->readBuf[0];
 }
 }  // namespace
 
@@ -410,15 +406,18 @@ uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const boo
     ctx.readBuf = fileReadBuffer;
     ctx.readBufSize = 1024;
 
-    if (!ctx.reader.init(true)) {
+    // One-shot mode: `data` holds the entire output, so back-references
-      LOG_ERR("ZIP", "Failed to init inflate reader");
+    // resolve inside it and no 32KB window is allocated.
+    InflateStream inflate;
+    if (!inflate.init(false)) {
+      LOG_ERR("ZIP", "Failed to init inflate stream");
       free(fileReadBuffer);
       free(data);
       return nullptr;
     }
-    ctx.reader.setReadCallback(zipReadCallback);
+    inflate.setFill(zipFillCallback, &ctx);
 
-    if (!ctx.reader.read(data, inflatedDataSize)) {
+    if (!inflate.read(data, inflatedDataSize)) {
       LOG_ERR("ZIP", "Failed to inflate file");
       free(fileReadBuffer);
       free(data);
@@ -501,20 +500,21 @@ bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t ch
     ctx.readBuf = fileReadBuffer;
     ctx.readBufSize = chunkSize;
 
-    if (!ctx.reader.init(true)) {
+    InflateStream inflate;
-      LOG_ERR("ZIP", "Failed to init inflate reader");
+    if (!inflate.init(true)) {
+      LOG_ERR("ZIP", "Failed to init inflate stream");
       free(outputBuffer);
       free(fileReadBuffer);
       return false;
     }
-    ctx.reader.setReadCallback(zipReadCallback);
+    inflate.setFill(zipFillCallback, &ctx);
 
     bool success = false;
     size_t totalProduced = 0;
 
     while (true) {
       size_t produced;
-      const InflateStatus status = ctx.reader.readAtMost(outputBuffer, chunkSize, &produced);
+      const InflateStream::Status status = inflate.readAtMost(outputBuffer, chunkSize, &produced);
 
       totalProduced += produced;
       if (totalProduced > static_cast<size_t>(inflatedDataSize)) {
@@ -530,7 +530,7 @@ bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t ch
         }
       }
 
-      if (status == InflateStatus::Done) {
+      if (status == InflateStream::Status::Done) {
         if (totalProduced != static_cast<size_t>(inflatedDataSize)) {
           LOG_ERR("ZIP", "Decompressed size mismatch (expected %zu, got %zu)", static_cast<size_t>(inflatedDataSize),
                   totalProduced);
@@ -541,16 +541,16 @@ bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t ch
         break;
       }
 
-      if (status == InflateStatus::Error) {
+      if (status == InflateStream::Status::Error) {
         LOG_ERR("ZIP", "Decompression failed");
         break;
       }
-      // InflateStatus::Ok: output buffer full, continue
+      // InflateStream::Status::Ok: output buffer full, continue
     }
 
     free(outputBuffer);
     free(fileReadBuffer);
-    return success;  // ctx.reader destructor frees the ring buffer
+    return success;  // inflate destructor frees the decompressor state + window
   }
 
   LOG_ERR("ZIP", "Unsupported compression method");

lib/hal/HalDisplay.cpp

@@ -77,6 +77,10 @@ void HalDisplay::deepSleep() { einkDisplay.deepSleep(); }
 
 uint8_t* HalDisplay::getFrameBuffer() const { return einkDisplay.getFrameBuffer(); }
 
+uint8_t* HalDisplay::lendFrameBufferStorage(uint32_t* sizeOut) { return einkDisplay.lendBuildStorage(sizeOut); }
+
+void HalDisplay::returnFrameBufferStorage() { einkDisplay.returnBuildStorage(); }
+
 void HalDisplay::copyGrayscaleBuffers(const uint8_t* lsbBuffer, const uint8_t* msbBuffer) {
   einkDisplay.copyGrayscaleBuffers(lsbBuffer, msbBuffer);
 }

lib/hal/HalDisplay.h

@@ -47,6 +47,14 @@ class HalDisplay {
   // Access to frame buffer
   uint8_t* getFrameBuffer() const;
 
+  // Lend the framebuffer's ~48 KB STORAGE to a memory-hungry phase (chapter
+  // builds) without freeing it: the allocation never moves, so repeated loans
+  // cannot fragment the heap (free+realloc measurably did). No display calls
+  // between lend and return; the panel keeps its last refreshed image. The
+  // buffer comes back white — redraw fully. Returns nullptr if already lent.
+  uint8_t* lendFrameBufferStorage(uint32_t* sizeOut);
+  void returnFrameBufferStorage();
+
   // X3 grayscale preconditioning (OEM "AA-pre-BW(mid)" settle pass), windowed
   // to the gray region in physical panel coordinates (no-arg = full frame).
   // Call after the BW base frame is displayed and before the grayscale planes

lib/miniz/library.json

@@ -0,0 +1,9 @@
+{
+  "name": "miniz",
+  "version": "11.3.2",
+  "description": "Vendored miniz (tinfl inflate only) + InflateStream wrapper",
+  "build": {
+    "srcDir": "src",
+    "includeDir": "src"
+  }
+}

lib/miniz/src/InflateStream.cpp

@@ -0,0 +1,165 @@
+#include "InflateStream.h"
+
+#include <BuildScratch.h>
+
+#include <cstdlib>
+#include <cstring>
+
+#include "MinizConfig.h"
+
+namespace {
+// tinfl's window must be a power of two; TINFL_LZ_DICT_SIZE is 32768.
+constexpr size_t WINDOW_SIZE = TINFL_LZ_DICT_SIZE;
+// tinfl_decompressor holds mz_uint32 arrays; 8 keeps the window aligned too.
+constexpr size_t STATE_ALIGNED = (sizeof(tinfl_decompressor) + 7) & ~size_t{7};
+}  // namespace
+
+InflateStream::~InflateStream() { deinit(); }
+
+bool InflateStream::init(const bool streaming) {
+  // Every consumer constructs a fresh stream per operation, so acquire storage
+  // from scratch each init (releasing any prior backing first).
+  deinit();
+
+  // During a framebuffer loan the lent 48KB is up for grabs: state (~11KB) +
+  // window (32KB) fit inside it, so a chapter-build inflate costs the heap
+  // nothing. Absent (or already claimed): plain heap, freed in deinit().
+  const size_t needed = STATE_ALIGNED + (streaming ? WINDOW_SIZE : 0);
+  arenaBase = buildscratch::claim(needed);
+  if (arenaBase) {
+    state = reinterpret_cast<tinfl_decompressor*>(arenaBase);
+    window = streaming ? arenaBase + STATE_ALIGNED : nullptr;
+  } else {
+    // Raw malloc (not makeUniqueNoThrow): the header keeps tinfl_decompressor
+    // an incomplete type so consumers never include miniz; both blocks are
+    // freed in deinit()/the destructor.
+    state = static_cast<tinfl_decompressor*>(malloc(sizeof(tinfl_decompressor)));
+    if (!state) return false;
+    if (streaming) {
+      window = static_cast<uint8_t*>(malloc(WINDOW_SIZE));
+      if (!window) return false;  // state kept; deinit()/next init reclaims it
+    }
+  }
+
+  tinfl_init(state);
+  windowPos = 0;
+  pendingStart = 0;
+  pendingLen = 0;
+  inPtr = nullptr;
+  inAvail = 0;
+  fill = nullptr;
+  fillCtx = nullptr;
+  inputExhausted = false;
+  zlibWrapped = false;
+  finished = false;
+  oneShotStart = nullptr;
+  return true;
+}
+
+void InflateStream::deinit() {
+  if (arenaBase) {
+    buildscratch::release(arenaBase);
+    arenaBase = nullptr;
+  } else {
+    free(state);
+    free(window);
+  }
+  state = nullptr;
+  window = nullptr;
+}
+
+void InflateStream::setSource(const uint8_t* src, const size_t len) {
+  inPtr = src;
+  inAvail = len;
+  inputExhausted = true;  // the whole input is present; nothing more will come
+}
+
+void InflateStream::setFill(const FillFn fn, void* ctx) {
+  fill = fn;
+  fillCtx = ctx;
+}
+
+InflateStream::Status InflateStream::readAtMost(uint8_t* dest, const size_t maxLen, size_t* produced) {
+  *produced = 0;
+  if (!state) return Status::Error;
+
+  const bool streaming = window != nullptr;
+  if (!streaming && !oneShotStart) oneShotStart = dest;
+
+  for (;;) {
+    // Drain window bytes left over from a previous tinfl call. In ring mode
+    // tinfl may produce more than the caller asked for in one shot -- the
+    // overshoot stays pending in the window until a later readAtMost.
+    if (pendingLen > 0) {
+      size_t n = maxLen - *produced;
+      if (n > pendingLen) n = pendingLen;
+      memcpy(dest + *produced, window + pendingStart, n);
+      pendingStart += n;
+      pendingLen -= n;
+      *produced += n;
+    }
+    if (*produced == maxLen) {
+      return (finished && pendingLen == 0) ? Status::Done : Status::Ok;
+    }
+    if (finished) return Status::Done;
+
+    if (inAvail == 0 && !inputExhausted && fill) {
+      inAvail = fill(fillCtx, &inPtr);
+      if (inAvail == 0) inputExhausted = true;
+    }
+
+    const mz_uint32 flags = (zlibWrapped ? TINFL_FLAG_PARSE_ZLIB_HEADER : 0) |
+                            (inputExhausted ? 0 : TINFL_FLAG_HAS_MORE_INPUT) |
+                            (streaming ? 0 : TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);
+
+    size_t inBytes = inAvail;
+    tinfl_status status;
+    size_t outBytes;
+    if (streaming) {
+      // Ring mode invariant: tinfl derives its wrap mask from
+      // (cursor offset + avail_out), so avail_out MUST always reach the end of
+      // the 32KB window -- never cap it to the caller's remaining space.
+      outBytes = WINDOW_SIZE - windowPos;
+      status = tinfl_decompress(state, inPtr, &inBytes, window, window + windowPos, &outBytes, flags);
+      pendingStart = windowPos;
+      pendingLen = outBytes;
+      windowPos += outBytes;
+      if (windowPos == WINDOW_SIZE) windowPos = 0;
+    } else {
+      // One-shot: back-references resolve directly inside the destination buffer.
+      outBytes = maxLen - *produced;
+      status = tinfl_decompress(state, inPtr, &inBytes, oneShotStart, dest + *produced, &outBytes, flags);
+      *produced += outBytes;
+    }
+    inPtr += inBytes;
+    inAvail -= inBytes;
+
+    if (status == TINFL_STATUS_DONE) {
+      finished = true;  // drain any pending window bytes on the next pass
+      continue;
+    }
+    if (status < TINFL_STATUS_DONE) return Status::Error;  // corrupt stream / adler mismatch
+    // TINFL_STATUS_NEEDS_MORE_INPUT loops back to the fill above; once the fill
+    // runs dry the HAS_MORE_INPUT flag drops and tinfl either finishes or fails
+    // (truncated stream) instead of spinning.
+    if (status == TINFL_STATUS_NEEDS_MORE_INPUT && inputExhausted && inAvail == 0) {
+      return Status::Error;
+    }
+    if (*produced == maxLen) {
+      return (finished && pendingLen == 0) ? Status::Done : Status::Ok;
+    }
+  }
+}
+
+bool InflateStream::read(uint8_t* dest, const size_t len) {
+  size_t total = 0;
+  while (total < len) {
+    size_t produced = 0;
+    const Status status = readAtMost(dest + total, len - total, &produced);
+    total += produced;
+    if (status == Status::Error) return false;
+    if (status == Status::Done) return total == len;
+    if (produced == 0) return false;  // no progress safeguard
+  }
+  return true;
+}

lib/miniz/src/InflateStream.h

@@ -0,0 +1,95 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// Forward declaration keeps miniz out of consumer translation units; the
+// decompressor state is heap-allocated in the .cpp where the type is complete.
+struct tinfl_decompressor_tag;
+
+// Streaming deflate decompressor wrapping miniz's tinfl.
+//
+// Replaces the uzlib-backed InflateReader on the throughput paths (EPUB zip
+// entries, PNG IDAT). tinfl decodes via lookup tables where uzlib walks the
+// Huffman tree bit-by-bit -- several times faster on this CPU -- at the cost
+// of a larger decompressor state (~11KB, transient for the scope of the
+// stream; taken from the lent framebuffer bytes via buildscratch::claim()
+// when a FrameBufferLoan is active, heap otherwise). FontDecompressor
+// intentionally stays on InflateReader:
+// its one-shot flash-resident group decompressions are tiny, and the render
+// path should not carry the extra state allocation.
+//
+// Two modes:
+//   init(false) -- one-shot: the destination buffer holds the ENTIRE output,
+//                  so back-references resolve inside it and no 32KB window is
+//                  allocated. read()/readAtMost() must be driven with
+//                  contiguous, forward-only slices of that one buffer
+//                  (a single read(dest, totalSize) is the common case).
+//   init(true)  -- streaming: allocates a 32KB window; output can go to any
+//                  buffer in any-sized chunks across calls.
+//
+// Input is either a single contiguous buffer (setSource) or pulled on demand
+// through a fill callback (setFill): return the number of bytes available and
+// point *data at them (valid until the next fill call); return 0 at end of
+// input. Call setZlibWrapped() before the first read when the stream has a
+// zlib header (e.g. PNG IDAT).
+class InflateStream {
+ public:
+  enum class Status {
+    Ok,     // Output buffer full; more decompressed data remains.
+    Done,   // Stream ended cleanly. produced may be < maxLen.
+    Error,  // Corrupt/truncated stream, or decompression failed.
+  };
+
+  using FillFn = size_t (*)(void* ctx, const uint8_t** data);
+
+  InflateStream() = default;
+  ~InflateStream();
+  InflateStream(const InflateStream&) = delete;
+  InflateStream& operator=(const InflateStream&) = delete;
+
+  // Allocate decompressor state (and the 32KB window when streaming) and reset
+  // stream state. Reuses existing allocations on repeated calls. Returns false
+  // on OOM.
+  bool init(bool streaming);
+
+  // Free the decompressor state and window.
+  void deinit();
+
+  // Provide the entire compressed input as one contiguous buffer.
+  void setSource(const uint8_t* src, size_t len);
+
+  // Provide compressed input on demand. ctx is passed back to fn verbatim.
+  void setFill(FillFn fn, void* ctx);
+
+  // Declare the input zlib-wrapped (2-byte header + trailing adler32).
+  void setZlibWrapped() { zlibWrapped = true; }
+
+  // Decompress exactly len bytes into dest. Returns false if the stream ends
+  // or errors before producing len bytes.
+  bool read(uint8_t* dest, size_t len);
+
+  // Decompress up to maxLen bytes into dest; *produced gets the byte count.
+  Status readAtMost(uint8_t* dest, size_t maxLen, size_t* produced);
+
+ private:
+  tinfl_decompressor_tag* state = nullptr;  // ~11KB: heap, or inside the claimed build scratch
+  uint8_t* window = nullptr;                // 32KB ring, streaming mode only
+  uint8_t* arenaBase = nullptr;             // non-null when state/window live in lent framebuffer bytes
+  size_t windowPos = 0;                     // ring write cursor
+  // Decompressed-but-undelivered region of the window (tinfl can overshoot the
+  // caller's requested length; the overshoot waits here for the next read).
+  size_t pendingStart = 0;
+  size_t pendingLen = 0;
+
+  const uint8_t* inPtr = nullptr;
+  size_t inAvail = 0;
+  FillFn fill = nullptr;
+  void* fillCtx = nullptr;
+  bool inputExhausted = false;
+  bool zlibWrapped = false;
+  bool finished = false;
+
+  // One-shot mode: tinfl needs the output buffer start for back-references.
+  uint8_t* oneShotStart = nullptr;
+};

lib/miniz/src/MinizConfig.h

@@ -0,0 +1,35 @@
+/* CrossPoint only needs miniz's low-level streaming inflate (tinfl). The
+ * archive, deflate, stdio, and zlib-compatibility layers are compiled out so
+ * the vendored library stays small and never touches the filesystem or clock.
+ * Include this header instead of <miniz.h> so every translation unit sees the
+ * same configuration. */
+#pragma once
+
+#define MINIZ_NO_STDIO
+#define MINIZ_NO_TIME
+#define MINIZ_NO_ARCHIVE_APIS
+#define MINIZ_NO_ARCHIVE_WRITING_APIS
+#define MINIZ_NO_DEFLATE_APIS
+#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES
+
+// The ESP32 mask ROM exports tinfl_* at fixed addresses via DIRECT linker
+// script assignments (e.g. "tinfl_decompress = 0x...;" in the ROM .ld),
+// which override object-file definitions -- without these renames the
+// firmware silently binds to the ROM's 2021 build (TINFL_LESS_MEMORY, a
+// different tinfl_decompressor layout) and corrupts inflate state on real
+// data. Rename so the linker can never capture them. The prefix is
+// crosspoint_ (NOT freeink_) so a future branch that links FreeInkBook's
+// identically-renamed copy does not collide.
+#define tinfl_decompress crosspoint_tinfl_decompress
+#define tinfl_decompress_mem_to_heap crosspoint_tinfl_decompress_mem_to_heap
+#define tinfl_decompress_mem_to_mem crosspoint_tinfl_decompress_mem_to_mem
+#define tinfl_decompress_mem_to_callback crosspoint_tinfl_decompress_mem_to_callback
+#define mz_crc32 crosspoint_mz_crc32
+#define mz_adler32 crosspoint_mz_adler32
+#define mz_free crosspoint_mz_free
+
+// Include the vendored miniz by relative path: ESP-IDF ships a ROM miniz.h
+// with the SAME include guard but a different (TINFL_LESS_MEMORY) struct
+// layout -- resolving <miniz.h> through the platform include path would
+// silently compile against the wrong structures.
+#include "../third_party/miniz.h"

lib/miniz/src/miniz_impl.c

@@ -0,0 +1,7 @@
+/* Compiles the vendored miniz with CrossPoint's configuration. The include
+ * order is load-bearing (the config defines/renames must be seen first). */
+// clang-format off
+#include "MinizConfig.h"
+
+#include "../third_party/miniz.c"
+// clang-format on

src/activities/reader/EpubReaderActivity.cpp

@@ -262,14 +262,42 @@ void EpubReaderActivity::loop() {
     return;
   }
 
+  // Lazily resume a partial's extension build once the reader nears its watermark. Far from
+  // it the rebuild is all cost (whole-chapter re-layout from page 0) and no benefit this
+  // session, so reopening a partial deliberately does NOT start it (see the deferral in
+  // render()); crossing this margin is the signal that the reader will actually need pages
+  // past the watermark soon. Uses the last render's viewport so pagination matches the
+  // partial being extended.
+  if (section && !section->isBuilding() && section->isPartial() && !RenderLock::peek() && buildViewportWidth > 0 &&
+      !partialRebuildStartFailed &&
+      section->currentPage + PARTIAL_REBUILD_START_MARGIN >= static_cast<int>(section->pageCount)) {
+    RenderLock lock;
+    if (!section->startBuild(SETTINGS.getReaderFontId(), SETTINGS.getReaderLineCompression(),
+                             SETTINGS.extraParagraphSpacing, SETTINGS.paragraphAlignment, buildViewportWidth,
+                             buildViewportHeight, SETTINGS.hyphenationEnabled, SETTINGS.embeddedStyle,
+                             SETTINGS.imageRendering, SETTINGS.focusReadingEnabled)) {
+      // Not fatal: the partial keeps serving its pages; crossing the watermark falls back to
+      // the blocking extension in render(). Don't retry every tick.
+      partialRebuildStartFailed = true;
+      LOG_ERR("ERS", "Failed to start deferred partial extension build");
+    } else {
+      LOG_DBG("ERS", "Reader near partial watermark (%d/%d), resuming extension build", section->currentPage,
+              section->pageCount);
+    }
+  }
+
   // Drive any in-progress incremental section build forward, off the page-turn critical path,
   // but only within a small window ahead of the reader: an unbounded build monopolized the
   // RenderLock and locked out page turns. The build follows the reader instead, and instant
   // reopen comes from suspendBuild() persisting the laid-out pages as a partial on exit.
   // Skip while the render mutex is busy so we never delay a pending render; re-check
   // isBuilding() under the lock since render() may have just finished it.
+  // While extending a partial (rebuild from a previous session), pageCount is pinned at the
+  // partial's watermark until the build catches up, so the window check would wrongly read
+  // "far enough ahead" and stall the build at 0 pages -- then the first turn past the
+  // watermark re-parses the whole chapter synchronously. Keep ticking until it finalizes.
   if (section && section->isBuilding() && !RenderLock::peek() &&
-      static_cast<int>(section->pageCount) < section->currentPage + BUILD_WINDOW_AHEAD) {
+      (section->isPartial() || static_cast<int>(section->pageCount) < section->currentPage + BUILD_WINDOW_AHEAD)) {
     RenderLock lock;
     // Re-check under the lock: render() (which also holds the RenderLock) may have finalized the
     // build between the outer isBuilding() check and acquiring the lock here, in which case
@@ -946,11 +974,17 @@ void EpubReaderActivity::render(RenderLock&& lock) {
 
   const uint16_t viewportWidth = renderer.getScreenWidth() - orientedMarginLeft - orientedMarginRight;
   const uint16_t viewportHeight = renderer.getScreenHeight() - orientedMarginTop - orientedMarginBottom;
+  // Capture for loop()'s lazy partial-extension start (must match this render's layout params).
+  buildViewportWidth = viewportWidth;
+  buildViewportHeight = viewportHeight;
 
   if (!section) {
     const auto filepath = epub->getSpineItem(currentSpineIndex).href;
     LOG_DBG("ERS", "Loading file: %s, index: %d", filepath.c_str(), currentSpineIndex);
     section = std::unique_ptr<Section>(new Section(epub, currentSpineIndex, renderer));
+    // Fresh section, fresh chance: a failed lazy extension start in a previous
+    // section must not suppress watermark-triggered rebuilds for this one.
+    partialRebuildStartFailed = false;
 
     // A finalized cache serves every page as-is. A partial cache (suspended build from a
     // previous session) serves its pages instantly too, but a build must still run to lay
@@ -993,16 +1027,25 @@ void EpubReaderActivity::render(RenderLock&& lock) {
         // The popup's own refresh is a plain FAST, so force the page that replaces it onto the HALF
         // ghost-cleanup path -- otherwise the "INDEXING" text ghosts under the rendered page.
         pagesUntilFullRefresh = 1;
-        const auto popupFn = [this]() { GUI.drawPopup(renderer, tr(STR_INDEXING)); };
+        // No popup redraws while the framebuffer is lent to the build below;
+        // the panel holds the popup displayed above (e-ink is persistent).
+        const auto popupFn = [this]() {
+          if (renderer.hasFrameBuffer()) GUI.drawPopup(renderer, tr(STR_INDEXING));
+        };
+        // Lend the framebuffer's 48 KB to the blocking full build; restored
+        // (white) at scope exit, and the page render below redraws everything.
+        GfxRenderer::FrameBufferLoan loan(renderer);
         if (!section->createSectionFile(SETTINGS.getReaderFontId(), SETTINGS.getReaderLineCompression(),
                                         SETTINGS.extraParagraphSpacing, SETTINGS.paragraphAlignment, viewportWidth,
                                         viewportHeight, SETTINGS.hyphenationEnabled, SETTINGS.embeddedStyle,
                                         SETTINGS.imageRendering, SETTINGS.focusReadingEnabled, popupFn)) {
           LOG_ERR("ERS", "Failed to persist page data to SD");
           section.reset();
+          loan.end();  // restore before anything draws
           showBuildError();
           return;
         }
+        loan.end();
       } else {
         // Lay out just enough to show the landing page; loop() builds the rest behind it. Show the
         // indexing popup up front only when the build will actually be slow: a large spine (its
@@ -1010,52 +1053,73 @@ void EpubReaderActivity::render(RenderLock&& lock) {
         // a deep resume/jump that must lay out many pages to reach the landing page. Tiny sections
         // build in a blink and stay popup-free.
         const int target = pendingPageJump.has_value() ? *pendingPageJump : (nextPageNumber < 0 ? 0 : nextPageNumber);
-        const size_t spineBytes = epub->getCumulativeSpineItemSize(currentSpineIndex) -
-                                  (currentSpineIndex > 0 ? epub->getCumulativeSpineItemSize(currentSpineIndex - 1) : 0);
-        // Popup only when the build will actually be slow: a big spine whose HTML still needs
-        // inflating (the multi-second cost), or a deep page target. A reopen with cached HTML builds
-        // fast, so no popup -- that's what made an already-indexed book look like it was reindexing.
-        // A partial cache that already covers the target page shows it instantly: never popup.
-        const bool willInflate = !section->hasHtmlCache();
         const bool anchorJump = !pendingAnchor.empty();
-        bool showPopup;
+
-        if (anchorJump) {
+        // Landing well inside a partial: the page (or anchor, via the on-disk map) is already
-          // An anchor jump's cost is bounded by the anchor's page, not `target`. An anchor already
+        // servable, so don't restart the extension build now -- it re-lays out the WHOLE chapter
-          // in the on-disk map (partial or finalized cache) lands instantly: no popup. Otherwise it
+        // from page 0 (minutes of background CPU + SD writes on a giant spine), pure waste when
-          // lies beyond the indexed watermark and the build may lay out the whole spine to find it,
+        // the reader never nears the watermark this session. loop() starts it lazily once the
-          // so gate on spine size alone -- laying out a big spine takes seconds even with cached
+        // reader is within PARTIAL_REBUILD_START_MARGIN pages of the watermark.
-          // HTML. Ordinary chapter-top TOC jumps resolve on page 0 and stay popup-free.
+        if (section->isPartial() &&
-          showPopup = !section->findAnchor(pendingAnchor).has_value() && spineBytes > BUILD_POPUP_BYTE_THRESHOLD;
+            (anchorJump ? section->getPageForAnchor(pendingAnchor).has_value()
+                        : target + PARTIAL_REBUILD_START_MARGIN < static_cast<int>(section->pageCount))) {
+          LOG_DBG("ERS", "Partial covers target %d of %d; deferring extension build", target, section->pageCount);
         } else {
-          const bool targetAvailable = target < static_cast<int>(section->pageCount);
+          const size_t spineBytes =
-          showPopup = !targetAvailable &&
+              epub->getCumulativeSpineItemSize(currentSpineIndex) -
-                      ((spineBytes > BUILD_POPUP_BYTE_THRESHOLD && willInflate) || target > BUILD_POPUP_PAGE_THRESHOLD);
+              (currentSpineIndex > 0 ? epub->getCumulativeSpineItemSize(currentSpineIndex - 1) : 0);
-        }
+          // Popup only when the build will actually be slow: a big spine whose HTML still needs
-        if (showPopup) {
+          // inflating (the multi-second cost), or a deep page target. A reopen with cached HTML builds
-          GUI.drawPopup(renderer, tr(STR_INDEXING));
+          // fast, so no popup -- that's what made an already-indexed book look like it was reindexing.
-          // HALF-clear the popup when the page replaces it, else "INDEXING" ghosts under the page.
+          // A partial cache that already covers the target page shows it instantly: never popup.
-          pagesUntilFullRefresh = 1;
+          const bool willInflate = !section->hasHtmlCache();
-        }
+          bool showPopup;
-        if (!section->startBuild(SETTINGS.getReaderFontId(), SETTINGS.getReaderLineCompression(),
+          if (anchorJump) {
-                                 SETTINGS.extraParagraphSpacing, SETTINGS.paragraphAlignment, viewportWidth,
+            // An anchor jump's cost is bounded by the anchor's page, not `target`. An anchor already
-                                 viewportHeight, SETTINGS.hyphenationEnabled, SETTINGS.embeddedStyle,
+            // in the on-disk map (partial or finalized cache) lands instantly: no popup. Otherwise it
-                                 SETTINGS.imageRendering, SETTINGS.focusReadingEnabled)) {
+            // lies beyond the indexed watermark and the build may lay out the whole spine to find it,
-          LOG_ERR("ERS", "Failed to start section build");
+            // so gate on spine size alone -- laying out a big spine takes seconds even with cached
-          section.reset();
+            // HTML. Ordinary chapter-top TOC jumps resolve on page 0 and stay popup-free.
-          showBuildError();
+            showPopup = !section->findAnchor(pendingAnchor).has_value() && spineBytes > BUILD_POPUP_BYTE_THRESHOLD;
-          return;
+          } else {
-        }
+            const bool targetAvailable = target < static_cast<int>(section->pageCount);
-        while (!section->isBuildComplete() &&
+            showPopup = !targetAvailable && ((spineBytes > BUILD_POPUP_BYTE_THRESHOLD && willInflate) ||
-               (anchorJump ? !section->findAnchor(pendingAnchor) : static_cast<int>(section->pageCount) <= target)) {
+                                             target > BUILD_POPUP_PAGE_THRESHOLD);
-          // Anchor jump: build until the anchor's page is laid out (usually page 0), checking a
+          }
-          // partial's on-disk anchor map too so an already-indexed anchor resolves immediately.
+          if (showPopup) {
-          // Otherwise: build until the target page exists. loop() builds the rest behind it.
+            GUI.drawPopup(renderer, tr(STR_INDEXING));
-          if (!section->buildSomeMore(BUILD_PAGES_PER_CHUNK)) {
+            // HALF-clear the popup when the page replaces it, else "INDEXING" ghosts under the page.
-            LOG_ERR("ERS", "Failed during incremental section build");
+            pagesUntilFullRefresh = 1;
+          }
+          // Lend the framebuffer's 48 KB to the blocking pre-render burst
+          // (startBuild inflates the whole spine HTML — the memory peak). The
+          // background buildSomeMore chunks in loop() do NOT get the loan: they
+          // deliberately interleave with page renders. Restored before render.
+          GfxRenderer::FrameBufferLoan loan(renderer);
+          if (!section->startBuild(SETTINGS.getReaderFontId(), SETTINGS.getReaderLineCompression(),
+                                   SETTINGS.extraParagraphSpacing, SETTINGS.paragraphAlignment, viewportWidth,
+                                   viewportHeight, SETTINGS.hyphenationEnabled, SETTINGS.embeddedStyle,
+                                   SETTINGS.imageRendering, SETTINGS.focusReadingEnabled)) {
+            LOG_ERR("ERS", "Failed to start section build");
             section.reset();
+            loan.end();  // restore before anything draws (showBuildError renders a popup)
             showBuildError();
             return;
           }
+          while (!section->isBuildComplete() &&
+                 (anchorJump ? !section->findAnchor(pendingAnchor) : static_cast<int>(section->pageCount) <= target)) {
+            // Anchor jump: build until the anchor's page is laid out (usually page 0), checking a
+            // partial's on-disk anchor map too so an already-indexed anchor resolves immediately.
+            // Otherwise: build until the target page exists. loop() builds the rest behind it.
+            if (!section->buildSomeMore(BUILD_PAGES_PER_CHUNK)) {
+              LOG_ERR("ERS", "Failed during incremental section build");
+              section.reset();
+              loan.end();  // restore before anything draws (showBuildError renders a popup)
+              showBuildError();
+              return;
+            }
+          }
+          loan.end();
         }
       }
     } else {
@@ -1098,6 +1162,16 @@ void EpubReaderActivity::render(RenderLock&& lock) {
   // Extend the build to the requested page if needed (for partials and in-progress builds).
   // This runs every render, so it covers both the first page and any forward turn that gets
   // ahead of the background builder; pages already built do no work here.
+  //
+  // Crossing a partial's watermark before the extension rebuild has caught up means a
+  // synchronous wait spanning the remaining prefix re-layout -- potentially tens of
+  // seconds on a giant spine. Show the indexing popup so it isn't a silent freeze
+  // (the page that replaces it takes the HALF ghost-cleanup path). Ordinary window
+  // catch-ups on a non-partial build are a page or two and stay popup-free.
+  if (section->isPartial() && section->currentPage >= static_cast<int>(section->pageCount)) {
+    GUI.drawPopup(renderer, tr(STR_INDEXING));
+    pagesUntilFullRefresh = 1;
+  }
   while (section->isPartial() && section->currentPage >= static_cast<int>(section->pageCount)) {
     // Start a build to extend a partial toward the requested page.
     if (!section->isBuilding() &&

src/activities/reader/EpubReaderActivity.h

@@ -63,6 +63,15 @@ class EpubReaderActivity final : public Activity {
   SavedPosition savedPositions[MAX_FOOTNOTE_DEPTH] = {};
   int footnoteDepth = 0;
 
+  // Viewport of the last render(), captured so loop()'s lazy partial-extension start
+  // builds with IDENTICAL layout parameters to the pages already rendered (a mismatch
+  // would paginate differently than the partial being extended). 0 = no render yet.
+  uint16_t buildViewportWidth = 0;
+  uint16_t buildViewportHeight = 0;
+  // Set when the lazy extension start failed, so loop() doesn't retry (and log) every
+  // tick; the blocking extension in render() remains the fallback past the watermark.
+  bool partialRebuildStartFailed = false;
+
   // Last position persisted by render()'s saveProgress, used to skip redundant
   // writeAtomic calls on no-op re-renders (menu/bookmark/screenshot).
   int lastSavedSpineIndex = -1;
@@ -85,6 +94,13 @@ class EpubReaderActivity final : public Activity {
   // in one sitting -- instant reopen comes from Section::suspendBuild() persisting the pages
   // already laid out as a partial file on exit/sleep.
   static constexpr int BUILD_WINDOW_AHEAD = 5;
+  // Reopening a partial does NOT immediately restart its extension build (a whole-chapter
+  // re-layout from page 0 -- minutes of background CPU + SD writes on a giant spine, wasted
+  // when the reader never crosses the watermark that session). Instead loop() starts it once
+  // the reader is within this many pages of the watermark: at ~30s per page read and ~100-300ms
+  // per page rebuilt, this margin gives the rebuild ample runway to catch up (and finalize)
+  // before the reader arrives.
+  static constexpr int PARTIAL_REBUILD_START_MARGIN = 15;
   // Show the indexing popup when an initial build must lay out more than this many pages up front
   // (a deep resume/jump into a not-yet-built section), so it isn't a silent wait. Kept independent
   // of the small look-ahead window so ordinary landings stay popup-free.
@@ -124,6 +140,11 @@ class EpubReaderActivity final : public Activity {
   void onExit() override;
   void loop() override;
   void render(RenderLock&& lock) override;
+  // Full CPU speed + fast loop ticks while a section build runs: at the low-power
+  // frequency a giant chapter's background rebuild stretches from ~40s to many
+  // minutes, so the reader exits before it can finalize and the next open restarts
+  // it from page 0. Reverts to normal power behavior the moment the build finishes.
+  bool skipLoopDelay() override { return section && section->isBuilding(); }
   bool isReaderActivity() const override { return true; }
   ScreenshotInfo getScreenshotInfo() const override;
   CrossPointPosition getCurrentPosition() const;

src/activities/reader/ReaderActivity.cpp

@@ -5,6 +5,8 @@
 #include <I18n.h>
 #include <Memory.h>
 
+#include <optional>
+
 #include "CrossPointSettings.h"
 #include "Epub.h"
 #include "EpubReaderActivity.h"
@@ -40,10 +42,20 @@ std::unique_ptr<Epub> ReaderActivity::loadEpub(const std::string& path) {
   // First open: building the spine/TOC index (book.bin) takes a couple of seconds. Show the
   // indexing popup so it isn't a silent wait on the home screen. The cachePath/hash is known at
   // construction, so this check is valid before load(); a cached open loads in a blink -> no popup.
-  if (!Storage.exists((epub->getCachePath() + "/book.bin").c_str())) {
+  const bool uncached = !Storage.exists((epub->getCachePath() + "/book.bin").c_str());
+  if (uncached) {
     GUI.drawPopup(renderer, tr(STR_INDEXING));
   }
-  if (epub->load(true, SETTINGS.embeddedStyle == 0)) {
+  bool loaded;
+  {
+    // Lend the framebuffer's 48 KB to the container parse (expat + spine/TOC
+    // build). The popup just displayed stays on the panel; whichever reader
+    // activity follows redraws the full screen anyway.
+    std::optional<GfxRenderer::FrameBufferLoan> loan;
+    if (uncached) loan.emplace(renderer);
+    loaded = epub->load(true, SETTINGS.embeddedStyle == 0);
+  }
+  if (loaded) {
     return epub;
   }