Add catalog mode for large EPUB containers

Implements a two-tier system for handling EPUB books: small books use in-RAM metadata (probe ladder with 48KB/72KB/96KB rungs), while large containers (e.g. webnovel omnibuses with 1500+ spine items) automatically fall back to SD-backed BookCatalog with persistent index. The catalog fast path reuses existing catalog.fibc files to skip probing on subsequent opens. Adds framebuffer borrowing for larger probe rungs and incremental chapter building for giant chapters.

src/activities/reader/BookPaginator.cpp

@@ -57,6 +57,7 @@ class ProgressSink : public freeink::book::PageSink {
 
 bool BookPaginator::open(const std::string& path, const std::string& cacheDir, GfxRenderer& renderer,
                          const bool forcePlainText) {
+  cacheDir_ = cacheDir;
   close();
 
   if (!source_.open(path.c_str())) {
@@ -68,7 +69,21 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G
   const size_t len = path.size();
   isTxt_ = forcePlainText || (len > 4 && strcasecmp(path.c_str() + len - 4, ".txt") == 0);
 
+  // The cache dir hosts both the page caches and (for omnibuses) the
+  // container index, so bind it before the catalog fast path below.
+  cache_.setDir(cacheDir.c_str());
+
   if (!isTxt_) {
+    bool fbLent = false;
+
+    // Catalog fast path: an existing catalog.fibc means this container
+    // already outgrew the in-RAM probe ladder on a previous open -- load the
+    // compact resident tables and skip the ladder entirely. A stale index
+    // (container changed) is removed inside openCatalog() and the ladder
+    // below decides afresh what the book now needs.
+    if (cache_.exists(freeink::book::BookCatalog::kCatalogName)) {
+      catalogMode_ = openCatalog();
+    }
     // The whole book budget must coexist with a ~100+ KB chapter-build
     // scratch on a heap where the reader sees ~140 KB free, so worst-case
     // arena sizing is unaffordable. Open in two passes: parse into a
@@ -84,82 +99,107 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G
     // temp arena, both proved to pin large blocks mid-heap and split the
     // region the chapter builds need.
     size_t bookUsed = 0;
-    {
+    if (!catalogMode_) {
+      // Probe ladder: most books fit the 48 KB rung; webnovel omnibuses
+      // (1500+ spine items) need far more container metadata, so the bigger
+      // rungs borrow the framebuffer (the LOADING popup already on the panel
+      // survives — e-ink holds its image without the buffer).
+      static constexpr size_t kProbeSizes[] = {kBookArenaSize, 72 * 1024, 96 * 1024};
+      BookStatus st = BookStatus::OutOfMemory;
+      for (size_t attempt = 0; attempt < sizeof(kProbeSizes) / sizeof(kProbeSizes[0]); ++attempt) {
+        const size_t probeSize = kProbeSizes[attempt];
+        if (attempt > 0 && !fbLent) {
+          renderer.releaseFrameBufferForBuild();
+          fbLent = true;
+        }
+        auto scratchBuf = makeUniqueNoThrow<uint8_t[]>(kOpenScratchSize);
+        auto tempBookBuf = makeUniqueNoThrow<uint8_t[]>(probeSize);
+        if (!scratchBuf || !tempBookBuf) {
+          LOG_ERR("FIB", "OOM: open working set (%u B, free heap %u)",
+                  static_cast<unsigned>(kOpenScratchSize + probeSize), static_cast<unsigned>(ESP.getFreeHeap()));
+          st = BookStatus::OutOfMemory;
+          break;  // the heap itself is short; a bigger rung cannot help
+        }
+        Arena scratch(scratchBuf.get(), kOpenScratchSize);
+        Arena tempArena(tempBookBuf.get(), probeSize);
+        freeink::book::Book probeBook;
+        st = probeBook.open(source_, tempArena, scratch);
+        if (st == BookStatus::Ok) {
+          bookUsed = tempArena.used();
+          if (attempt > 0) {
+            LOG_INF("FIB", "Large container: book arena needs %u B (attempt %u)", static_cast<unsigned>(bookUsed),
+                    static_cast<unsigned>(attempt + 1));
+          }
+          break;
+        }
+        if (st != BookStatus::OutOfMemory) break;  // real parse error, not arena size
+        LOG_DBG("FIB", "Book probe outgrew %u B, retrying larger", static_cast<unsigned>(probeSize));
+      }
+      if (st == BookStatus::OutOfMemory) {
+        // Even the 96 KB rung overflowed: webnovel-omnibus territory (1,700+
+        // spine items need ~400 KB of container metadata). Build the
+        // SD-backed catalog instead -- compact resident tables, everything
+        // string-shaped stays on the card. The framebuffer is already lent
+        // unless the heap itself was short, in which case the catalog build
+        // fails with its own clear log.
+        if (!fbLent) {
+          renderer.releaseFrameBufferForBuild();
+          fbLent = true;
+        }
+        LOG_INF("FIB", "Container outgrew the probe ladder; building SD catalog");
+        catalogMode_ = buildCatalog() && openCatalog();
+        if (catalogMode_) st = BookStatus::Ok;
+      }
+      if (st != BookStatus::Ok) {
+        LOG_ERR("FIB", "Book open failed: %d (%s)", static_cast<int>(st), path.c_str());
+        if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart();
+        close();
+        return false;
+      }
+    }
+
+    if (!catalogMode_) {
+      // PHASE B: exact-size arena into the emptied heap, re-parse for keeps.
+      bookBuf_ = makeUniqueNoThrow<uint8_t[]>(bookUsed + 128);
       auto scratchBuf = makeUniqueNoThrow<uint8_t[]>(kOpenScratchSize);
-      auto tempBookBuf = makeUniqueNoThrow<uint8_t[]>(kBookArenaSize);
-      if (!scratchBuf || !tempBookBuf) {
-        LOG_ERR("FIB", "OOM: open working set (%u B, free heap %u)",
-                static_cast<unsigned>(kOpenScratchSize + kBookArenaSize), static_cast<unsigned>(ESP.getFreeHeap()));
+      if (!bookBuf_ || !scratchBuf) {
+        LOG_ERR("FIB", "OOM: book arena reopen (%u B)", static_cast<unsigned>(bookUsed + 128));
+        if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart();
         close();
         return false;
       }
       Arena scratch(scratchBuf.get(), kOpenScratchSize);
-      Arena tempArena(tempBookBuf.get(), kBookArenaSize);
-      freeink::book::Book probeBook;
-      const BookStatus st = probeBook.open(source_, tempArena, scratch);
+      bookArena_.init(bookBuf_.get(), bookUsed + 128);
+      const BookStatus st = book_.open(source_, bookArena_, scratch);
       if (st != BookStatus::Ok) {
-        LOG_ERR("FIB", "Book open failed: %d (%s)", static_cast<int>(st), path.c_str());
+        LOG_ERR("FIB", "Exact-size reopen failed: %d", static_cast<int>(st));
+        if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart();
         close();
         return false;
       }
-      bookUsed = tempArena.used();
-    }
-
-    // PHASE B: exact-size arena into the emptied heap, re-parse for keeps.
-    bookBuf_ = makeUniqueNoThrow<uint8_t[]>(bookUsed + 128);
-    auto scratchBuf = makeUniqueNoThrow<uint8_t[]>(kOpenScratchSize);
-    if (!bookBuf_ || !scratchBuf) {
-      LOG_ERR("FIB", "OOM: book arena reopen (%u B)", static_cast<unsigned>(bookUsed + 128));
-      close();
-      return false;
-    }
-    Arena scratch(scratchBuf.get(), kOpenScratchSize);
-    bookArena_.init(bookBuf_.get(), bookUsed + 128);
-    const BookStatus st = book_.open(source_, bookArena_, scratch);
-    if (st != BookStatus::Ok) {
-      LOG_ERR("FIB", "Exact-size reopen failed: %d", static_cast<int>(st));
-      close();
-      return false;
-    }
-
-    // Stylesheet: build in a temporary working arena, keep only the rules.
-    auto tempSheetBuf = makeUniqueNoThrow<uint8_t[]>(kSheetArenaSize);
-    if (tempSheetBuf) {
-      Arena sheetArena(tempSheetBuf.get(), kSheetArenaSize);
-      CssStylesheetBuilder builder;
-      if (builder.begin(sheetArena)) {
-        for (size_t m = 0; m < book_.manifestCount(); ++m) {
-          const ManifestItem* item = book_.manifestItem(m);
-          if (item == nullptr || item->mediaType == nullptr || strcmp(item->mediaType, "text/css") != 0) continue;
-          if (const freeink::book::ZipEntry* e = book_.zip().find(item->href)) {
-            builder.addSheet(source_, *e, scratch);
-          }
-        }
-        sheet_ = builder.finish();
-        if (builder.skippedSheets() > 0) {
-          LOG_INF("FIB", "%u stylesheet(s) skipped (size cap)", builder.skippedSheets());
-        }
-        if (sheet_.ruleCount > 0) {
-          // CssRule is self-contained (name hashes + POD declaration), so the
-          // rule array can move out of the 12 KB working arena wholesale.
-          const size_t rulesBytes = sheet_.ruleCount * sizeof(freeink::book::CssRule);
-          sheetBuf_ = makeUniqueNoThrow<uint8_t[]>(rulesBytes);
-          if (sheetBuf_) {
-            memcpy(sheetBuf_.get(), sheet_.rules, rulesBytes);
-            sheet_.rules = reinterpret_cast<const freeink::book::CssRule*>(sheetBuf_.get());
-          } else {
-            sheet_ = freeink::book::CssStylesheet{};  // OOM: lay out with element defaults
-          }
-        }
+      buildStylesheet(scratch);
+      scratchBuf.reset();  // free the reopen scratch before the framebuffer returns
+    } else if (catalog_.cssCount() > 0) {
+      // Catalog mode: CSS entries were resolved at catalog open; the build
+      // only needs a transient inflate-capable scratch.
+      auto scratchBuf = makeUniqueNoThrow<uint8_t[]>(kOpenScratchSize);
+      if (scratchBuf) {
+        Arena scratch(scratchBuf.get(), kOpenScratchSize);
+        buildStylesheet(scratch);
+      } else {
+        LOG_ERR("FIB", "OOM: stylesheet scratch; using element defaults");
       }
     }
 
-    LOG_INF("FIB", "Book open: %u spine items, book arena %u B (exact), %u CSS rules, free heap %u",
-            static_cast<unsigned>(book_.spineCount()), static_cast<unsigned>(bookUsed), sheet_.ruleCount,
-            static_cast<unsigned>(ESP.getFreeHeap()));
-  }
+    if (fbLent && !renderer.restoreFrameBufferAfterBuild()) {
+      LOG_ERR("FIB", "Framebuffer restore failed after large open");
+      ESP.restart();  // transients freed; a failed 48 KB realloc means heap corruption
+    }
 
-  cache_.setDir(cacheDir.c_str());
+    LOG_INF("FIB", "Book open%s: %u spine items, book arena %u B, %u CSS rules, free heap %u",
+            catalogMode_ ? " (SD catalog)" : "", static_cast<unsigned>(spineCount()),
+            static_cast<unsigned>(bookArena_.used()), sheet_.ruleCount, static_cast<unsigned>(ESP.getFreeHeap()));
+  }
 
   if (!buildFontChain(renderer)) {
     close();
@@ -174,10 +214,128 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G
   return true;
 }
 
+// Loads an existing catalog.fibc into an exactly-sized resident arena. A
+// stale index (container changed since the build) is deleted so the caller
+// can rebuild; any other failure just reports false.
+bool BookPaginator::openCatalog() {
+  size_t resident = 0;
+  BookStatus st = freeink::book::BookCatalog::residentBytes(cache_, &resident);
+  if (st != BookStatus::Ok) return false;
+
+  bookBuf_ = makeUniqueNoThrow<uint8_t[]>(resident);
+  auto scratchBuf = makeUniqueNoThrow<uint8_t[]>(4096);  // fingerprint scan buffer
+  if (!bookBuf_ || !scratchBuf) {
+    LOG_ERR("FIB", "OOM: catalog resident arena (%u B)", static_cast<unsigned>(resident));
+    bookBuf_.reset();
+    return false;
+  }
+  bookArena_.init(bookBuf_.get(), resident);
+  Arena scratch(scratchBuf.get(), 4096);
+  st = catalog_.open(source_, cache_, bookArena_, scratch);
+  if (st == BookStatus::Stale) {
+    LOG_INF("FIB", "Catalog stale (container changed); rebuilding");
+    cache_.remove(freeink::book::BookCatalog::kCatalogName);
+    bookBuf_.reset();
+    return false;
+  }
+  if (st != BookStatus::Ok) {
+    LOG_ERR("FIB", "Catalog open failed: %d", static_cast<int>(st));
+    bookBuf_.reset();
+    return false;
+  }
+  LOG_INF("FIB", "SD catalog open: %u spines, %u toc, resident %u B", static_cast<unsigned>(catalog_.spineCount()),
+          static_cast<unsigned>(catalog_.tocCount()), static_cast<unsigned>(bookArena_.used()));
+  return true;
+}
+
+// Streams the container index to SD. Caller has the framebuffer lent; the
+// two arenas mirror the chapter-build split (record tables vs parse stream).
+// Measured for a 1,732-spine omnibus: records ~72.5 KB, parse ~45.3 KB.
+bool BookPaginator::buildCatalog() {
+  constexpr size_t kRecordsIdeal = 96 * 1024;
+  constexpr size_t kRecordsFloor = 76 * 1024;
+  constexpr size_t kParseSize = 50 * 1024;
+  constexpr size_t kAllocSlack = 64;
+
+  auto parseBuf = makeUniqueNoThrow<uint8_t[]>(kParseSize);
+  if (!parseBuf) {
+    LOG_ERR("FIB", "OOM: catalog parse arena (free heap %u)", static_cast<unsigned>(ESP.getFreeHeap()));
+    return false;
+  }
+  const size_t block = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT);
+  const size_t recordsSize = std::min(kRecordsIdeal, block > kAllocSlack ? block - kAllocSlack : 0);
+  std::unique_ptr<uint8_t[]> recordsBuf;
+  if (recordsSize >= kRecordsFloor) recordsBuf = makeUniqueNoThrow<uint8_t[]>(recordsSize);
+  if (!recordsBuf) {
+    LOG_ERR("FIB", "OOM: catalog record arena (want %u B, max block %u)", static_cast<unsigned>(recordsSize),
+            static_cast<unsigned>(block));
+    return false;
+  }
+  Arena records(recordsBuf.get(), recordsSize);
+  Arena parse(parseBuf.get(), kParseSize);
+
+  const uint32_t t0 = millis();
+  const BookStatus st = freeink::book::BookCatalog::build(source_, cache_, records, &parse);
+  if (st != BookStatus::Ok) {
+    LOG_ERR("FIB", "Catalog build failed: %d (records %u/%u refused %u, parse %u/%u refused %u B)",
+            static_cast<int>(st), static_cast<unsigned>(records.highWater()), static_cast<unsigned>(recordsSize),
+            static_cast<unsigned>(records.failedAllocSize()), static_cast<unsigned>(parse.highWater()),
+            static_cast<unsigned>(kParseSize), static_cast<unsigned>(parse.failedAllocSize()));
+    return false;
+  }
+  LOG_INF("FIB", "Catalog built in %ums (records %u/%u, parse %u/%u B)", static_cast<unsigned>(millis() - t0),
+          static_cast<unsigned>(records.highWater()), static_cast<unsigned>(recordsSize),
+          static_cast<unsigned>(parse.highWater()), static_cast<unsigned>(kParseSize));
+  return true;
+}
+
+// Builds the book stylesheet in a temporary working arena and keeps only the
+// compacted rule array. CSS items come from whichever catalog is active.
+void BookPaginator::buildStylesheet(Arena& scratch) {
+  auto tempSheetBuf = makeUniqueNoThrow<uint8_t[]>(kSheetArenaSize);
+  if (!tempSheetBuf) return;
+  Arena sheetArena(tempSheetBuf.get(), kSheetArenaSize);
+  CssStylesheetBuilder builder;
+  if (!builder.begin(sheetArena)) return;
+  if (catalogMode_) {
+    for (size_t c = 0; c < catalog_.cssCount(); ++c) {
+      builder.addSheet(source_, *catalog_.cssEntry(c), scratch);
+    }
+  } else {
+    for (size_t m = 0; m < book_.manifestCount(); ++m) {
+      const ManifestItem* item = book_.manifestItem(m);
+      if (item == nullptr || item->mediaType == nullptr || strcmp(item->mediaType, "text/css") != 0) continue;
+      if (const freeink::book::ZipEntry* e = book_.zip().find(item->href)) {
+        builder.addSheet(source_, *e, scratch);
+      }
+    }
+  }
+  sheet_ = builder.finish();
+  if (builder.skippedSheets() > 0) {
+    LOG_INF("FIB", "%u stylesheet(s) skipped (size cap)", builder.skippedSheets());
+  }
+  if (sheet_.ruleCount > 0) {
+    // CssRule is self-contained (name hashes + POD declaration), so the
+    // rule array can move out of the 12 KB working arena wholesale.
+    const size_t rulesBytes = sheet_.ruleCount * sizeof(freeink::book::CssRule);
+    sheetBuf_ = makeUniqueNoThrow<uint8_t[]>(rulesBytes);
+    if (sheetBuf_) {
+      memcpy(sheetBuf_.get(), sheet_.rules, rulesBytes);
+      sheet_.rules = reinterpret_cast<const freeink::book::CssRule*>(sheetBuf_.get());
+    } else {
+      sheet_ = freeink::book::CssStylesheet{};  // OOM: lay out with element defaults
+    }
+  }
+}
+
 void BookPaginator::close() {
+  suspendBuild();  // commits a partial so the next open resumes instantly
+  chapterSource_.close();
   source_.close();
   reader_ = freeink::book::PageCacheReader();
   book_ = freeink::book::Book();
+  catalog_ = freeink::book::BookCatalog();
+  catalogMode_ = false;
   sheet_ = freeink::book::CssStylesheet{};
   chain_ = freeink::book::FontChain();
   hyphBlob_.reset();
@@ -192,9 +350,8 @@ void BookPaginator::close() {
 }
 
 const char* BookPaginator::language() const {
-  if (!isTxt_ && book_.metadata().language != nullptr && book_.metadata().language[0] != '\0') {
-    return book_.metadata().language;
-  }
+  const char* lang = catalogMode_ ? catalog_.metadata().language : book_.metadata().language;
+  if (!isTxt_ && lang != nullptr && lang[0] != '\0') return lang;
   return "en";
 }
 
@@ -340,26 +497,82 @@ bool BookPaginator::isChapterCached(const uint16_t spineIndex) {
   return cache_.exists(name);
 }
 
-freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex, const BuildProgress& progress) {
+freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex, const BuildProgress& progress,
+                                                       const uint32_t targetChar, const float targetFraction) {
   const uint32_t gen = generation();
   if (!freeink::book::pageCacheName(spineIndex, gen, cacheName_, sizeof(cacheName_))) {
     return BookStatus::Unsupported;
   }
 
+  // An in-progress incremental build of this same chapter/generation stays
+  // live across ensureChapter calls (page turns re-enter here).
+  if (building_) {
+    if (buildGeneration_ == gen && curSpine_ == spineIndex) return BookStatus::Ok;
+    suspendBuild();  // settings/spine changed under the build
+  }
+
   curSpine_ = kNoSpine;
+  partialReaderOpen_ = false;
   if (!indexBuf_ && !reallocChapterArenas()) return BookStatus::OutOfMemory;
   indexArena_.reset();
   BookStatus st = reader_.open(cache_, cacheName_, gen, indexArena_);
-  if (st == BookStatus::Ok) {
+  if (st == BookStatus::Ok && !reader_.isPartial()) {
     curSpine_ = spineIndex;
     return st;
   }
 
-  const ManifestItem* item = isTxt_ ? nullptr : book_.spineItem(spineIndex);
-  const freeink::book::ZipEntry* entry = (!isTxt_ && item != nullptr) ? book_.zip().find(item->href) : nullptr;
-  if (!isTxt_ && entry == nullptr) return BookStatus::NotFound;
+  // Resolve the chapter's entry + href into MEMBERS: in catalog mode the
+  // ZipEntry is materialized from the SD index (no arena-resident copy
+  // exists), and the incremental session retains pointers across steps.
+  const freeink::book::ZipEntry* entry = nullptr;
+  if (!isTxt_) {
+    if (catalogMode_) {
+      if (catalog_.spineEntry(spineIndex, &curEntry_) != BookStatus::Ok ||
+          catalog_.spineHref(spineIndex, chapterHref_, sizeof(chapterHref_)) != BookStatus::Ok) {
+        return BookStatus::NotFound;
+      }
+    } else {
+      const ManifestItem* item = book_.spineItem(spineIndex);
+      const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr;
+      if (e == nullptr) return BookStatus::NotFound;
+      curEntry_ = *e;
+      snprintf(chapterHref_, sizeof(chapterHref_), "%s", item->href);
+    }
+    entry = &curEntry_;
+  }
   if (isTxt_ && spineIndex != 0) return BookStatus::NotFound;
 
+  // A suspended partial serves its pages instantly while a fresh background
+  // rebuild re-lays the chapter from the start (deterministic layout makes
+  // the rebuilt prefix identical). The partial file stays readable during
+  // the rebuild because writes stream to a temp name until commit.
+  const bool incrementalLanding = targetChar != kBuildAll || targetFraction >= 0.0f;
+  if (st == BookStatus::Ok && reader_.isPartial() && !isTxt_ && incrementalLanding) {
+    partialReaderOpen_ = true;
+    const BookStatus inc = startIncremental(spineIndex, *entry, targetChar, targetFraction, progress);
+    if (inc == BookStatus::Ok) {
+      curSpine_ = spineIndex;
+      return BookStatus::Ok;
+    }
+    // Incremental could not start (extraction/OOM): fall through to the
+    // blocking build below, which replaces the partial outright.
+    partialReaderOpen_ = false;
+  }
+
+  // Giant uncached chapter with a known landing position: build only a
+  // little past it and finish behind the reader.
+  constexpr uint32_t kIncrementalThreshold = 96 * 1024;  // uncompressed bytes
+  if (!isTxt_ && incrementalLanding && entry->uncompressedSize > kIncrementalThreshold) {
+    reader_ = freeink::book::PageCacheReader();
+    indexBuf_.reset();  // nothing to serve from it; reclaim 12 KB for the session
+    const BookStatus inc = startIncremental(spineIndex, *entry, targetChar, targetFraction, progress);
+    if (inc == BookStatus::Ok) {
+      curSpine_ = spineIndex;
+      return BookStatus::Ok;
+    }
+    // fall through to the blocking build
+  }
+
   // The pagination working set lives only for this call, as TWO arenas:
   // layout buffers + the writer's index in one, the parse stream (inflate
   // window + decompressor + XML chunks) in the other. Splitting kills the
@@ -380,11 +593,10 @@ freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex
   // reserve is being squeezed — raise it first.
   constexpr size_t kParserHeapReserve = 22 * 1024;
   constexpr size_t kLayoutIdeal = 60 * 1024;
-  // Floor is measured-demand-plus-margin, not comfort: heaviest observed
-  // chapter (device, incl. chunked writer index) is 44,964 B. Raising this
-  // makes builds fail upfront on heap shapes where they would have fit —
-  // a 46 KiB floor already rejected a 46,452 B offer that would have built.
-  constexpr size_t kLayoutFloor = 45 * 1024;
+  // Floor is measured-demand-plus-margin, not comfort. The LayoutEngine
+  // itself now lives in this arena (session refactor moved it off the
+  // stack): small-profile host high water incl. writer index is 48,656 B.
+  constexpr size_t kLayoutFloor = 48 * 1024;
   constexpr size_t kParseIdeal = 50 * 1024;
   constexpr size_t kParseFloor = 46 * 1024;
   constexpr size_t kParseStored = 8 * 1024;  // stored entries skip inflate state
@@ -456,7 +668,7 @@ freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex
     ProgressSink sink(writer, progress);
     uint32_t totalChars = 0;
     st = isTxt_ ? ChapterLayout::layoutPlainText(source_, params_, scratch, sink, nullptr, &totalChars)
-                : ChapterLayout::layout(source_, book_.zip(), *entry, item->href, params_, scratch, sink, nullptr,
+                : ChapterLayout::layout(source_, zip(), *entry, chapterHref_, params_, scratch, sink, nullptr,
                                         &totalChars, &parseArena);
     writer.setTotalChars(totalChars);
     if (st == BookStatus::Ok && !writer.finish()) st = BookStatus::IoError;
@@ -485,9 +697,264 @@ freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex
   return st;
 }
 
+// Extracts (inflates) a deflated chapter to <cacheDir>/xNNNN.raw once, so the
+// layout session parses a stored file (~8 KB resident parse state instead of
+// the ~46 KB inflate stream). The file is keyed on the spine index only — it
+// survives settings changes (generations) and is reused by rebuilds.
+bool BookPaginator::extractChapter(const uint16_t spineIndex, const freeink::book::ZipEntry& entry) {
+  char rawName[16];
+  snprintf(rawName, sizeof(rawName), "x%04u.raw", spineIndex);
+  char rawPath[128];
+  snprintf(rawPath, sizeof(rawPath), "%s/%s", cacheDir_.c_str(), rawName);
+
+  chapterSource_.close();
+  if (!Storage.exists(rawPath) || (chapterSource_.open(rawPath) && chapterSource_.size() != entry.uncompressedSize)) {
+    chapterSource_.close();
+    // Inflate the whole entry to SD. Transient: one inflate stream (~46 KB).
+    auto scratchBuf = makeUniqueNoThrow<uint8_t[]>(kOpenScratchSize);
+    if (!scratchBuf) {
+      LOG_ERR("FIB", "OOM: chapter extract scratch");
+      return false;
+    }
+    Arena scratch(scratchBuf.get(), kOpenScratchSize);
+    freeink::book::ZipEntryReader zr;
+    if (zr.open(source_, entry, scratch) != BookStatus::Ok) return false;
+    uint8_t* buf = static_cast<uint8_t*>(scratch.alloc(4096, 1));
+    if (buf == nullptr) return false;
+    char tmpPath[136];
+    snprintf(tmpPath, sizeof(tmpPath), "%s.tmp", rawPath);
+    {
+      HalFile out;
+      if (!Storage.openFileForWrite("FIB", tmpPath, out)) return false;
+      for (;;) {
+        const int32_t n = zr.read(buf, 4096);
+        if (n < 0) {
+          Storage.remove(tmpPath);
+          return false;
+        }
+        if (n == 0) break;
+        if (out.write(buf, n) != static_cast<size_t>(n)) {
+          Storage.remove(tmpPath);
+          return false;
+        }
+      }
+    }
+    Storage.remove(rawPath);
+    if (!Storage.rename(tmpPath, rawPath)) return false;
+    if (!chapterSource_.open(rawPath)) return false;
+  }
+  rawEntry_ = freeink::book::ZipEntryReader::rawEntry(entry.uncompressedSize);
+  return true;
+}
+
+freeink::book::BookStatus BookPaginator::startIncremental(const uint16_t spineIndex,
+                                                          const freeink::book::ZipEntry& entry,
+                                                          const uint32_t targetChar, const float targetFraction,
+                                                          const BuildProgress& progress) {
+  // Resident while reading: layout (buffers + engine + writer index chunks —
+  // a ~700-page chapter needs ~6 chunks) + a SMALL parse arena (the chapter
+  // is stored/extracted, so no inflate window). The image pre-scan's probe
+  // inflate (~46 KB) runs in a TRANSIENT arena freed right after begin() —
+  // keeping it out of the resident set is what lets the session coexist
+  // with the framebuffer and font caches on the C3. Expat pools stay on the
+  // system heap (~19 KB while the session lives).
+  constexpr size_t kBuildLayoutSize = 56 * 1024;
+  constexpr size_t kBuildParseSize = 12 * 1024;
+  constexpr size_t kPrescanSize = 46 * 1024;
+  constexpr uint32_t kWindowPages = 2;  // build this far past the landing page
+
+  // ensureChapter resolved curEntry_/chapterHref_ before calling here; the
+  // `entry` argument aliases curEntry_ (a member -- the session's readers
+  // retain pointers into it across steps).
+  freeink::book::BookSource* chapterSrc = &source_;
+  const freeink::book::ZipEntry* parseEntry = &entry;
+  if (entry.method != 0) {
+    if (!extractChapter(spineIndex, entry)) return BookStatus::IoError;
+    chapterSrc = &chapterSource_;
+    parseEntry = &rawEntry_;
+  }
+
+  buildLayoutBuf_ = makeUniqueNoThrow<uint8_t[]>(kBuildLayoutSize);
+  buildParseBuf_ = makeUniqueNoThrow<uint8_t[]>(kBuildParseSize);
+  if (!buildLayoutBuf_ || !buildParseBuf_) {
+    LOG_ERR("FIB", "OOM: incremental build arenas (free heap %u)", static_cast<unsigned>(ESP.getFreeHeap()));
+    buildLayoutBuf_.reset();
+    buildParseBuf_.reset();
+    return BookStatus::OutOfMemory;
+  }
+  buildLayoutArena_.init(buildLayoutBuf_.get(), kBuildLayoutSize);
+  buildParseArena_.init(buildParseBuf_.get(), kBuildParseSize);
+
+  buildWriter_ = freeink::book::PageCacheWriter();
+  const uint32_t gen = generation();
+  if (!buildWriter_.begin(cache_, cacheName_, gen, buildLayoutArena_)) {
+    buildLayoutBuf_.reset();
+    buildParseBuf_.reset();
+    return BookStatus::IoError;
+  }
+  BookStatus st;
+  {
+    // Transient probe arena for begin() only (image dimension pre-scan).
+    auto prescanBuf = makeUniqueNoThrow<uint8_t[]>(kPrescanSize);
+    Arena prescanArena(prescanBuf.get(), prescanBuf ? kPrescanSize : 0);
+    st = buildSession_.begin(source_, &zip(), *chapterSrc, *parseEntry, chapterHref_, params_, buildLayoutArena_,
+                             buildWriter_, &buildParseArena_, prescanBuf ? &prescanArena : nullptr);
+  }
+  if (st != BookStatus::Ok) {
+    LOG_ERR("FIB", "Incremental begin failed: %d", static_cast<int>(st));
+    buildSession_.abort();
+    buildWriter_ = freeink::book::PageCacheWriter();
+    buildLayoutBuf_.reset();
+    buildParseBuf_.reset();
+    return st;
+  }
+  buildGeneration_ = gen;
+  building_ = true;
+
+  // Synchronous burst: lay out just past the landing position. For a char
+  // target, "the target's page + window exists" is the stop (pageForChar is
+  // a watermark). For a fraction target the stop is the parsed BYTE ratio —
+  // the caller then resolves its landing char against chars-so-far, which by
+  // construction sits right at this stop point.
+  const uint32_t t0 = millis();
+  while (!buildSession_.done()) {
+    if (targetFraction >= 0.0f) {
+      const uint64_t total = buildSession_.bytesTotal();
+      if (total > 0 &&
+          buildSession_.bytesConsumed() >= static_cast<uint64_t>(targetFraction * static_cast<float>(total)) &&
+          buildWriter_.pageCount() > kWindowPages) {
+        break;
+      }
+    } else if (buildWriter_.pageCount() > buildWriter_.pageForChar(targetChar) + kWindowPages) {
+      break;
+    }
+    st = buildSession_.step(4);
+    if (st != BookStatus::Ok) {
+      LOG_ERR("FIB", "Incremental build failed: %d (layout %u/%u refused %u, parse %u/%u refused %u B)",
+              static_cast<int>(st), static_cast<unsigned>(buildLayoutArena_.highWater()),
+              static_cast<unsigned>(kBuildLayoutSize), static_cast<unsigned>(buildLayoutArena_.failedAllocSize()),
+              static_cast<unsigned>(buildParseArena_.highWater()), static_cast<unsigned>(kBuildParseSize),
+              static_cast<unsigned>(buildParseArena_.failedAllocSize()));
+      suspendBuild();
+      return st;
+    }
+    if (progress.fn != nullptr) progress.fn(progress.ctx, buildWriter_.pageCount());
+  }
+  if (buildSession_.done()) return finalizeBuild();
+  LOG_INF("FIB", "Chapter %u building incrementally: %u pages to target in %ums (of ~%u est)", spineIndex,
+          buildWriter_.pageCount(), static_cast<unsigned>(millis() - t0), static_cast<unsigned>(estimatedTotalPages()));
+  return BookStatus::Ok;
+}
+
+freeink::book::BookStatus BookPaginator::pumpBuild(const uint32_t pages) {
+  if (!building_) return BookStatus::Ok;
+  const BookStatus st = buildSession_.step(pages);
+  if (st != BookStatus::Ok) {
+    LOG_ERR("FIB", "Incremental pump failed: %d", static_cast<int>(st));
+    suspendBuild();
+    return st;
+  }
+  if (buildSession_.done()) return finalizeBuild();
+  return BookStatus::Ok;
+}
+
+freeink::book::BookStatus BookPaginator::finalizeBuild() {
+  buildWriter_.setTotalChars(buildSession_.totalChars());
+  const bool committed = buildWriter_.finish();
+  const uint32_t pages = buildWriter_.pageCount();
+  buildSession_.abort();
+  buildWriter_ = freeink::book::PageCacheWriter();
+  buildLayoutBuf_.reset();
+  buildParseBuf_.reset();
+  building_ = false;
+  partialReaderOpen_ = false;
+
+  reader_ = freeink::book::PageCacheReader();
+  if (!indexBuf_ && !reallocChapterArenas()) return BookStatus::OutOfMemory;
+  indexArena_.reset();
+  if (!committed) return BookStatus::IoError;
+  const BookStatus st = reader_.open(cache_, cacheName_, buildGeneration_, indexArena_);
+  LOG_INF("FIB", "Incremental build finalized: %u pages (open %d, free heap %u)", pages, static_cast<int>(st),
+          static_cast<unsigned>(ESP.getFreeHeap()));
+  return st;
+}
+
+void BookPaginator::suspendBuild() {
+  if (!building_) return;
+  buildWriter_.setTotalChars(buildSession_.totalChars());  // chars-so-far watermark
+  buildWriter_.suspend(static_cast<uint32_t>(buildSession_.bytesConsumed()),
+                       static_cast<uint32_t>(buildSession_.bytesTotal()));
+  buildSession_.abort();
+  buildWriter_ = freeink::book::PageCacheWriter();
+  buildLayoutBuf_.reset();
+  buildParseBuf_.reset();
+  building_ = false;
+  partialReaderOpen_ = false;
+  curSpine_ = kNoSpine;  // force a clean reopen (of the partial) next time
+}
+
+uint32_t BookPaginator::pageCount() const {
+  if (!building_) return reader_.pageCount();
+  const uint32_t w = buildWriter_.pageCount();
+  const uint32_t r = partialReaderOpen_ ? reader_.pageCount() : 0;
+  return w > r ? w : r;
+}
+
+uint32_t BookPaginator::totalChars() const {
+  if (!building_) return reader_.totalChars();
+  // A chars-so-far watermark: the denominator grows as the build advances,
+  // exactly like the legacy incremental build. Display code uses
+  // estimatedTotalPages() instead.
+  const uint32_t w = buildSession_.totalChars();
+  const uint32_t r = partialReaderOpen_ ? reader_.totalChars() : 0;
+  return w > r ? w : r;
+}
+
+uint32_t BookPaginator::charStartOfPage(const uint32_t pageIndex) const {
+  if (!building_) return reader_.charStart(pageIndex);
+  if (pageIndex < buildWriter_.pageCount()) return buildWriter_.charStart(pageIndex);
+  return partialReaderOpen_ ? reader_.charStart(pageIndex) : 0;
+}
+
+uint32_t BookPaginator::pageForChar(const uint32_t charOffset) const {
+  if (!building_) return reader_.pageForChar(charOffset);
+  // The rebuilt prefix is byte-identical to the partial's, so the indexes
+  // agree wherever they overlap — just ask the source that has more pages.
+  if (partialReaderOpen_ && reader_.pageCount() > buildWriter_.pageCount()) {
+    return reader_.pageForChar(charOffset);
+  }
+  return buildWriter_.pageForChar(charOffset);
+}
+
+uint32_t BookPaginator::estimatedTotalPages() const {
+  const uint32_t known = pageCount();
+  uint64_t consumed = 0;
+  uint64_t total = 0;
+  uint64_t basePages = 0;
+  if (building_) {
+    consumed = buildSession_.bytesConsumed();
+    total = buildSession_.bytesTotal();
+    basePages = buildWriter_.pageCount();
+  } else if (reader_.isPartial()) {
+    consumed = reader_.buildBytesConsumed();
+    total = reader_.buildBytesTotal();
+    basePages = reader_.pageCount();
+  } else {
+    return known;
+  }
+  if (consumed == 0 || total == 0 || basePages == 0) return known;
+  const uint64_t est = basePages * total / consumed;
+  return est > known ? static_cast<uint32_t>(est) : known;
+}
+
 bool BookPaginator::charForAnchor(const char* fragment, uint32_t* charOut) const {
   if (fragment == nullptr || fragment[0] == '\0') return false;
-  return reader_.charForAnchor(ZipCatalog::hashPath(fragment), charOut);
+  const uint32_t hash = ZipCatalog::hashPath(fragment);
+  if (building_) {
+    if (buildWriter_.charForAnchor(hash, charOut)) return true;
+    return partialReaderOpen_ && reader_.charForAnchor(hash, charOut);
+  }
+  return reader_.charForAnchor(hash, charOut);
 }
 
 freeink::book::BookStatus BookPaginator::readPage(const uint32_t pageIndex, freeink::book::Page* out) {
@@ -495,11 +962,19 @@ freeink::book::BookStatus BookPaginator::readPage(const uint32_t pageIndex, free
   // a failed reallocation (the arena would point at freed memory).
   if (curSpine_ == kNoSpine || !pageBuf_) return BookStatus::NotFound;
   pageArena_.reset();
+  if (building_) {
+    if (pageIndex < buildWriter_.pageCount()) return buildWriter_.readPage(pageIndex, pageArena_, out);
+    if (partialReaderOpen_ && pageIndex < reader_.pageCount()) {
+      return reader_.readPage(pageIndex, pageArena_, out);
+    }
+    return BookStatus::NotFound;  // beyond the watermark — pump and retry
+  }
   return reader_.readPage(pageIndex, pageArena_, out);
 }
 
 int BookPaginator::spineIndexForHref(const char* href) const {
   if (href == nullptr || href[0] == '\0' || isTxt_) return -1;
+  if (catalogMode_) return catalog_.spineIndexForHref(href);
   for (size_t s = 0; s < book_.spineCount(); ++s) {
     const ManifestItem* item = book_.spineItem(s);
     if (item != nullptr && strcmp(item->href, href) == 0) return static_cast<int>(s);
@@ -507,9 +982,36 @@ int BookPaginator::spineIndexForHref(const char* href) const {
   return -1;
 }
 
+bool BookPaginator::spineZipEntry(const int spineIndex, freeink::book::ZipEntry* out) const {
+  if (isTxt_ || spineIndex < 0) return false;
+  if (catalogMode_) {
+    return catalog_.spineEntry(static_cast<size_t>(spineIndex), out) == BookStatus::Ok;
+  }
+  const ManifestItem* item = book_.spineItem(static_cast<size_t>(spineIndex));
+  const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr;
+  if (e == nullptr) return false;
+  *out = *e;
+  return true;
+}
+
 BookPaginator::TocItem BookPaginator::tocItem(const size_t index) const {
   TocItem out{"", nullptr, -1, 0};
-  const freeink::book::TocEntry* entry = isTxt_ ? nullptr : book_.tocEntry(index);
+  if (isTxt_) return out;
+  if (catalogMode_) {
+    // Title/fragment are SD reads into the shared member buffers -- valid
+    // until the next tocItem() call (see the TocItem declaration).
+    freeink::book::BookCatalog::TocItem item;
+    if (catalog_.tocItem(index, &item, tocTitleBuf_, sizeof(tocTitleBuf_), tocFragBuf_, sizeof(tocFragBuf_)) !=
+        BookStatus::Ok) {
+      return out;
+    }
+    out.title = tocTitleBuf_;
+    out.fragment = item.hasFragment ? tocFragBuf_ : nullptr;
+    out.spineIndex = item.spineIndex;
+    out.depth = item.depth;
+    return out;
+  }
+  const freeink::book::TocEntry* entry = book_.tocEntry(index);
   if (entry == nullptr) return out;
   out.title = entry->title;
   out.fragment = entry->fragment;
@@ -521,6 +1023,7 @@ BookPaginator::TocItem BookPaginator::tocItem(const size_t index) const {
 int BookPaginator::tocIndexForSpine(const int spineIndex) const {
   // The chapter's title is the last TOC entry at or before this spine item
   // (a spine item without its own entry belongs to the preceding heading).
+  if (catalogMode_) return catalog_.tocIndexForSpine(spineIndex);
   int best = -1;
   int bestSpine = -1;
   for (size_t t = 0; t < tocCount(); ++t) {
@@ -534,7 +1037,14 @@ int BookPaginator::tocIndexForSpine(const int spineIndex) const {
   return best;
 }
 
-// Spine weights use the uncompressed sizes already in the ZIP catalog — the
+uint32_t BookPaginator::spineSizeAt(const size_t spineIndex) const {
+  if (catalogMode_) return catalog_.spineSize(spineIndex);
+  const ManifestItem* item = book_.spineItem(spineIndex);
+  const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr;
+  return e != nullptr ? e->uncompressedSize : 0;
+}
+
+// Spine weights use the uncompressed sizes already in the catalog -- the
 // same "bigger chapters cover more of the book" heuristic the legacy engine
 // used, with zero extra state.
 float BookPaginator::bookProgress(const int spineIndex, const float chapterFraction) const {
@@ -542,10 +1052,8 @@ float BookPaginator::bookProgress(const int spineIndex, const float chapterFract
   uint64_t before = 0;
   uint64_t current = 0;
   uint64_t total = 0;
-  for (size_t s = 0; s < book_.spineCount(); ++s) {
-    const ManifestItem* item = book_.spineItem(s);
-    const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr;
-    const uint32_t size = e != nullptr ? e->uncompressedSize : 0;
+  for (size_t s = 0; s < spineCount(); ++s) {
+    const uint32_t size = spineSizeAt(s);
     if (static_cast<int>(s) < spineIndex) before += size;
     if (static_cast<int>(s) == spineIndex) current = size;
     total += size;
@@ -557,24 +1065,20 @@ float BookPaginator::bookProgress(const int spineIndex, const float chapterFract
 
 int BookPaginator::spineForBookFraction(const float bookFraction, float* chapterFractionOut) const {
   if (chapterFractionOut != nullptr) *chapterFractionOut = 0.0f;
-  if (isTxt_ || book_.spineCount() == 0) {
+  if (isTxt_ || spineCount() == 0) {
     if (chapterFractionOut != nullptr) *chapterFractionOut = bookFraction;
     return 0;
   }
   uint64_t total = 0;
-  for (size_t s = 0; s < book_.spineCount(); ++s) {
-    const ManifestItem* item = book_.spineItem(s);
-    const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr;
-    total += e != nullptr ? e->uncompressedSize : 0;
+  for (size_t s = 0; s < spineCount(); ++s) {
+    total += spineSizeAt(s);
   }
   const float f = bookFraction < 0.0f ? 0.0f : (bookFraction > 1.0f ? 1.0f : bookFraction);
   const uint64_t target = static_cast<uint64_t>(f * static_cast<float>(total));
   uint64_t cumulative = 0;
-  for (size_t s = 0; s < book_.spineCount(); ++s) {
-    const ManifestItem* item = book_.spineItem(s);
-    const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr;
-    const uint32_t size = e != nullptr ? e->uncompressedSize : 0;
-    if (target < cumulative + size || s + 1 == book_.spineCount()) {
+  for (size_t s = 0; s < spineCount(); ++s) {
+    const uint32_t size = spineSizeAt(s);
+    if (target < cumulative + size || s + 1 == spineCount()) {
       if (chapterFractionOut != nullptr && size > 0) {
         *chapterFractionOut = static_cast<float>(target - cumulative) / static_cast<float>(size);
       }

src/activities/reader/BookPaginator.h

@@ -20,6 +20,7 @@
 //                 produce one contiguous block
 // Steady-state page turns touch only the page arena (~2 KB used).
 
+#include <BookCatalog.h>
 #include <FreeInkBook.h>
 #include <cache/PageCache.h>
 #include <css/Css.h>
@@ -58,21 +59,40 @@ class BookPaginator {
   bool isOpen() const { return open_; }
   bool isTxt() const { return isTxt_; }
 
-  freeink::book::Book& book() { return book_; }
   freeink::book::BookSource* bookSource() { return &source_; }
-  size_t spineCount() const { return isTxt_ ? 1 : book_.spineCount(); }
+  // Container lookups (image probes, internal links) -- the in-RAM catalog or
+  // the SD-backed one, whichever this book opened with.
+  const freeink::book::ZipCatalog& zip() const { return catalogMode_ ? catalog_.zip() : book_.zip(); }
+  // True when the book outgrew the in-RAM probe ladder and runs on the
+  // SD-backed BookCatalog (webnovel omnibuses).
+  bool isCatalogMode() const { return catalogMode_; }
+  size_t spineCount() const {
+    if (isTxt_) return 1;
+    return catalogMode_ ? catalog_.spineCount() : book_.spineCount();
+  }
   const char* language() const;
-  const char* title() const { return isTxt_ ? "" : book_.metadata().title; }
-  const char* author() const { return isTxt_ ? "" : book_.metadata().author; }
+  const char* title() const {
+    return isTxt_ ? "" : (catalogMode_ ? catalog_.metadata().title : book_.metadata().title);
+  }
+  const char* author() const {
+    return isTxt_ ? "" : (catalogMode_ ? catalog_.metadata().author : book_.metadata().author);
+  }
+  // ZipEntry of a spine item (KOSync/xpath bridges). False when absent.
+  bool spineZipEntry(int spineIndex, freeink::book::ZipEntry* out) const;
 
   // --- TOC (flattened, resolved to spine indices) --------------------------
   struct TocItem {
+    // In catalog mode both strings live in a shared internal buffer that the
+    // NEXT tocItem() call overwrites -- use or copy them before iterating on.
     const char* title;
     const char* fragment;  // anchor within the chapter, or nullptr
     int spineIndex;        // -1 when the href is not a spine item
     uint8_t depth;
   };
-  size_t tocCount() const { return isTxt_ ? 0 : book_.tocCount(); }
+  size_t tocCount() const {
+    if (isTxt_) return 0;
+    return catalogMode_ ? catalog_.tocCount() : book_.tocCount();
+  }
   TocItem tocItem(size_t index) const;
   // First TOC entry pointing at `spineIndex` or an earlier chapter (the
   // chapter's display title); -1 when the TOC has no such entry.
@@ -100,14 +120,41 @@ class BookPaginator {
 
   // Opens the chapter's page cache for the current generation, laying the
   // chapter out first when missing or stale. Heavy only on that miss.
-  freeink::book::BookStatus ensureChapter(uint16_t spineIndex, const BuildProgress& progress = BuildProgress());
+  //
+  // `targetChar` — the landing position (chapter character offset), when the
+  // caller knows it. For a GIANT uncached chapter (or a suspended partial)
+  // this switches to the INCREMENTAL path: the chapter builds only a little
+  // past the target, isBuilding() turns true, pages serve from the partial
+  // build, and the caller finishes the rest via pumpBuild() between page
+  // turns. kBuildAll (the default) forces the classic blocking full build —
+  // required for percent jumps (they need the final page count).
+  // `targetFraction` (0..1) is the fraction-of-chapter alternative for
+  // landings that predate exact charStart positions (migrated legacy
+  // progress, whole-book percent jumps): the incremental build runs until
+  // the parsed BYTE ratio passes it — bytes track extracted characters
+  // closely enough that the subsequent pageForChar lands within a page.
+  static constexpr uint32_t kBuildAll = 0xFFFFFFFF;
+  freeink::book::BookStatus ensureChapter(uint16_t spineIndex, const BuildProgress& progress = BuildProgress(),
+                                          uint32_t targetChar = kBuildAll, float targetFraction = -1.0f);
   bool chapterReady() const { return curSpine_ != kNoSpine; }
   uint16_t currentSpine() const { return curSpine_; }
 
-  uint32_t pageCount() const { return reader_.pageCount(); }
-  uint32_t totalChars() const { return reader_.totalChars(); }
-  uint32_t charStartOfPage(uint32_t pageIndex) const { return reader_.charStart(pageIndex); }
-  uint32_t pageForChar(uint32_t charOffset) const { return reader_.pageForChar(charOffset); }
+  // --- incremental build (giant single-spine chapters) ---------------------
+  bool isBuilding() const { return building_; }
+  // Lays out up to `pages` more pages of the in-progress build; finalizes
+  // (writer commit + cache reopen) when the chapter ends. Callers gate this
+  // on a read-ahead window (currentPage + N) — never pump unboundedly.
+  freeink::book::BookStatus pumpBuild(uint32_t pages);
+  // Byte-ratio estimate of the final page count while building ("page X of
+  // ~Y"); the exact count once the build (or cache) is complete.
+  uint32_t estimatedTotalPages() const;
+
+  // While building incrementally these merge the live writer (rebuild
+  // watermark) with any reopened partial — see the .cpp implementations.
+  uint32_t pageCount() const;
+  uint32_t totalChars() const;
+  uint32_t charStartOfPage(uint32_t pageIndex) const;
+  uint32_t pageForChar(uint32_t charOffset) const;
   // Resolve an id="" fragment in the CURRENT chapter to a char offset.
   bool charForAnchor(const char* fragment, uint32_t* charOut) const;
 
@@ -143,10 +190,33 @@ class BookPaginator {
   bool reallocChapterArenas();
   void loadHyphenator();
   uint32_t fontFingerprint() const;
+  // Uncompressed bytes of a spine item -- the whole-book progress weights.
+  uint32_t spineSizeAt(size_t spineIndex) const;
+  // SD-backed catalog path (omnibuses whose container outgrows the probe
+  // ladder): open an existing catalog.fibc / build one. openCatalog removes
+  // a stale index (changed container) and returns false so the caller can
+  // rebuild; buildCatalog expects the framebuffer already lent.
+  bool openCatalog();
+  bool buildCatalog();
+  // Builds + compacts the book stylesheet (CSS items from whichever catalog).
+  void buildStylesheet(freeink::book::Arena& scratch);
+
+  // Incremental-build internals. A "giant" chapter (uncompressed size above
+  // kIncrementalThreshold) is extracted once to <cacheDir>/xNNNN.raw so the
+  // resident parse state is the stored-entry ~8 KB plus probe headroom, then
+  // laid out through a ChapterLayoutSession feeding buildWriter_. Session
+  // arenas (buildLayoutBuf_/buildParseBuf_) stay allocated while reading;
+  // suspendBuild() commits a partial cache on exit/spine-switch.
+  freeink::book::BookStatus startIncremental(uint16_t spineIndex, const freeink::book::ZipEntry& entry,
+                                             uint32_t targetChar, float targetFraction, const BuildProgress& progress);
+  freeink::book::BookStatus finalizeBuild();
+  void suspendBuild();
+  bool extractChapter(uint16_t spineIndex, const freeink::book::ZipEntry& entry);
 
   SdBookSource source_;
   SdCacheStorage cache_;
   freeink::book::Book book_;
+  freeink::book::BookCatalog catalog_;  // SD-backed container index (omnibuses)
   freeink::book::LayoutParams params_;
   freeink::book::PageCacheReader reader_;
   freeink::book::FontChain chain_;
@@ -176,4 +246,28 @@ class BookPaginator {
   uint16_t curSpine_ = kNoSpine;
   bool open_ = false;
   bool isTxt_ = false;
+  bool catalogMode_ = false;
+  std::string cacheDir_;  // per-book cache directory (extraction files live here)
+
+  // Current chapter identity. curEntry_ must be a MEMBER: the incremental
+  // layout session's ZipEntryReader retains a pointer to it across steps, and
+  // in catalog mode there is no arena-resident entry to point at.
+  freeink::book::ZipEntry curEntry_{};
+  char chapterHref_[512] = "";
+  // tocItem() string backing in catalog mode (single slot, see TocItem note).
+  mutable char tocTitleBuf_[256];
+  mutable char tocFragBuf_[256];
+
+  // Incremental-build session state (live only while building_).
+  freeink::book::ChapterLayoutSession buildSession_;
+  freeink::book::PageCacheWriter buildWriter_;
+  SdBookSource chapterSource_;                 // the extracted raw chapter file
+  freeink::book::ZipEntry rawEntry_{};         // headerless entry over chapterSource_
+  std::unique_ptr<uint8_t[]> buildLayoutBuf_;  // session layout arena backing
+  std::unique_ptr<uint8_t[]> buildParseBuf_;   // session parse arena backing
+  freeink::book::Arena buildLayoutArena_;
+  freeink::book::Arena buildParseArena_;
+  uint32_t buildGeneration_ = 0;  // generation the session was started under
+  bool building_ = false;
+  bool partialReaderOpen_ = false;  // reader_ holds a partial while rebuilding
 };

src/activities/reader/EpubReaderActivity.cpp

@@ -202,7 +202,7 @@ float EpubReaderActivity::currentBookFraction() const {
 
 void EpubReaderActivity::openReaderMenu() {
   const int currentPageDisplay = paginator.chapterReady() ? static_cast<int>(currentPage) + 1 : 0;
-  const int totalPages = paginator.chapterReady() ? static_cast<int>(paginator.pageCount()) : 0;
+  const int totalPages = paginator.chapterReady() ? static_cast<int>(paginator.estimatedTotalPages()) : 0;
   const int bookProgressPercent = clampPercent(static_cast<int>(currentBookFraction() * 100.0f + 0.5f));
   startActivityForResult(
       std::make_unique<EpubReaderMenuActivity>(renderer, mappedInput, paginator.title(), currentPageDisplay, totalPages,
@@ -251,6 +251,19 @@ void EpubReaderActivity::loop() {
     pendingReadFolderMove = false;
   }
 
+  // Incremental chapter build: keep a small window laid out ahead of the
+  // reader, in short bursts, only when no render is in flight. NEVER pump
+  // unboundedly — an ungated background build once locked the legacy reader
+  // solid. A chapter bigger than the window simply stays partial (suspended
+  // to disk on exit) and follows the reader; a small remainder finalizes.
+  if (paginator.isBuilding() && !RenderLock::peek()) {
+    constexpr uint32_t kWindowAhead = 5;
+    if (paginator.pageCount() < static_cast<uint32_t>(currentPage) + 1 + kWindowAhead) {
+      RenderLock lock;
+      if (paginator.isBuilding()) paginator.pumpBuild(2);
+    }
+  }
+
   if (automaticPageTurnActive) {
     if (mappedInput.wasReleased(MappedInputManager::Button::Confirm) ||
         mappedInput.wasReleased(MappedInputManager::Button::Back)) {
@@ -562,11 +575,10 @@ bool EpubReaderActivity::launchKOReaderSync() {
   // the whole-book percentage is the robust fallback.
   std::string localXPath = syntheticXPath(currentSpineIndex);
   if (!paginator.isTxt()) {
-    const freeink::book::ManifestItem* item = paginator.book().spineItem(currentSpineIndex);
-    const freeink::book::ZipEntry* entry = item != nullptr ? paginator.book().zip().find(item->href) : nullptr;
-    if (entry != nullptr) {
-      localXPath = BookXPath::xpathForCharStart(*paginator.bookSource(), paginator.book().zip(), *entry,
-                                                currentSpineIndex, lastCharStart);
+    freeink::book::ZipEntry entry;
+    if (paginator.spineZipEntry(currentSpineIndex, &entry)) {
+      localXPath = BookXPath::xpathForCharStart(*paginator.bookSource(), paginator.zip(), entry, currentSpineIndex,
+                                                lastCharStart);
     }
   }
   SavedProgressPosition localKoPos{std::move(localXPath), currentBookFraction()};
@@ -625,7 +637,9 @@ void EpubReaderActivity::toggleAutoPageTurn(const uint8_t selectedPageTurnOption
 
 void EpubReaderActivity::pageTurn(bool isForwardTurn) {
   if (isForwardTurn) {
-    if (paginator.chapterReady() && currentPage + 1 < paginator.pageCount()) {
+    // While a chapter is still building, pageCount() is a watermark, not the
+    // end of the chapter — keep advancing; render() builds the page in.
+    if (paginator.chapterReady() && (currentPage + 1 < paginator.pageCount() || paginator.isBuilding())) {
       currentPage++;
     } else {
       currentSpineIndex++;  // spineCount == end-of-book screen
@@ -691,7 +705,23 @@ bool EpubReaderActivity::ensureChapterAndPosition() {
       }
     };
 
-    const auto status = paginator.ensureChapter(static_cast<uint16_t>(currentSpineIndex), progressCb);
+    // Landing target: a known charStart (resume, reanchor, footnote return),
+    // a chapter fraction (migrated legacy progress, percent jumps), or the
+    // chapter top (plain page turns) all go INCREMENTAL on giant chapters —
+    // the build runs just past the landing point and finishes behind the
+    // reader. Only anchors (need the full anchor map) and last-page entries
+    // (need the final count) take the classic blocking build (kBuildAll).
+    uint32_t targetChar = BookPaginator::kBuildAll;
+    float targetFraction = -1.0f;
+    if (pendingCharStart.has_value()) {
+      targetChar = *pendingCharStart;
+    } else if (pendingChapterFraction.has_value()) {
+      targetFraction = *pendingChapterFraction;
+    } else if (pendingAnchor.empty() && !pendingLastPage) {
+      targetChar = 0;  // entering at the chapter top
+    }
+    const auto status =
+        paginator.ensureChapter(static_cast<uint16_t>(currentSpineIndex), progressCb, targetChar, targetFraction);
     if (!renderer.hasFrameBuffer() && !renderer.restoreFrameBufferAfterBuild()) {
       // Unrecoverable: nothing can be drawn again. The build arenas are freed
       // (ensureChapter released them), so this realloc failing means the heap
@@ -727,9 +757,22 @@ bool EpubReaderActivity::ensureChapterAndPosition() {
     pendingChapterFraction.reset();
     pendingLastPage = false;
   } else if (pendingChapterFraction.has_value()) {
-    const uint32_t targetChar =
-        static_cast<uint32_t>(*pendingChapterFraction * static_cast<float>(paginator.totalChars()));
-    currentPage = paginator.pageForChar(targetChar);
+    if (paginator.isBuilding()) {
+      // totalChars() is a build watermark here: fraction x watermark would
+      // land at fraction-of-the-PREFIX (half the intended depth for a 50%
+      // resume). Resolve against the estimated total page count instead;
+      // the build stopped right at the fraction's byte offset, so the
+      // clamped watermark page IS the landing page.
+      const uint32_t est = paginator.estimatedTotalPages();
+      const uint32_t built = paginator.pageCount();
+      uint32_t page = static_cast<uint32_t>(*pendingChapterFraction * static_cast<float>(est));
+      if (page >= built) page = built > 0 ? built - 1 : 0;
+      currentPage = page;
+    } else {
+      const uint32_t targetChar =
+          static_cast<uint32_t>(*pendingChapterFraction * static_cast<float>(paginator.totalChars()));
+      currentPage = paginator.pageForChar(targetChar);
+    }
     pendingChapterFraction.reset();
     pendingLastPage = false;
   } else if (pendingLastPage) {
@@ -737,8 +780,22 @@ bool EpubReaderActivity::ensureChapterAndPosition() {
     pendingLastPage = false;
   }
 
+  // Reaching past the last built page of an in-progress build means "keep
+  // building" (handled in render), not "clamp": pageCount() is a watermark.
+  if (paginator.isBuilding() && currentPage >= paginator.pageCount()) {
+    constexpr uint32_t kCatchUpBurst = 4;
+    if (!buildPopupShown && currentPage >= paginator.pageCount() + 2) {
+      // A deep jump has real work ahead of it; the burst below blocks.
+      GUI.drawPopup(renderer, tr(STR_INDEXING));
+      pagesUntilFullRefresh = 1;
+      buildPopupShown = true;
+    }
+    while (paginator.isBuilding() && currentPage >= paginator.pageCount()) {
+      if (paginator.pumpBuild(kCatchUpBurst) != freeink::book::BookStatus::Ok) break;
+    }
+  }
   if (paginator.pageCount() > 0 && currentPage >= paginator.pageCount()) {
-    currentPage = paginator.pageCount() - 1;
+    currentPage = paginator.pageCount() - 1;  // true end of chapter (or build failure)
   }
   return true;
 }
@@ -959,7 +1016,10 @@ void EpubReaderActivity::renderPage(const freeink::book::Page& page, int) {
 
 void EpubReaderActivity::renderStatusBar() const {
   const int currentPageDisplay = static_cast<int>(currentPage) + 1;
-  const float pageCount = paginator.chapterReady() ? static_cast<float>(paginator.pageCount()) : 0.0f;
+  // estimatedTotalPages() equals pageCount() once the chapter is fully
+  // built; while building it is the byte-ratio estimate, so "page X of Y"
+  // does not display the watermark as if it were the chapter total.
+  const float pageCount = paginator.chapterReady() ? static_cast<float>(paginator.estimatedTotalPages()) : 0.0f;
   const float bookProgress = currentBookFraction() * 100.0f;
 
   std::string title;

src/activities/reader/FreeInkBookStorage.h

@@ -91,6 +91,23 @@ class SdCacheStorage : public freeink::book::CacheStorage {
     return true;
   }
 
+  // Mid-build page read-back (incremental layout): the write handle is
+  // opened O_RDWR by SDCardManager, so seek-read-seek-back on the SAME
+  // handle is coherent — the exact pattern the legacy engine used. The
+  // write cursor MUST be restored or subsequent writes corrupt the cache;
+  // if that seek fails, fail loudly.
+  int32_t readBackAt(uint32_t offset, void* dst, uint32_t len) override {
+    if (!write_.isOpen()) return -1;
+    const size_t writePos = write_.position();
+    if (!write_.seekSet(offset)) return -1;
+    const int32_t n = write_.read(dst, len);
+    if (!write_.seekSet(writePos)) {
+      LOG_ERR("FIBCACHE", "readBackAt: write cursor restore FAILED");
+      return -1;
+    }
+    return n;
+  }
+
  private:
   static constexpr const char* kTempName = "_tmp.fibp";
 

src/activities/reader/FreeInkPageRenderer.cpp

@@ -100,8 +100,8 @@ bool ensureImageCached(BookPaginator& paginator, const std::string& cacheDir, co
                              static_cast<uint8_t>(img.height & 0xFF), static_cast<uint8_t>(img.height >> 8)};
   writer.failed = writer.file.write(header, sizeof(header)) != sizeof(header);
 
-  const freeink::book::BookStatus st = freeink::book::ImageRenderer::render(
-      *paginator.bookSource(), paginator.book().zip(), img, scratch, &G2Writer::onRow, &writer);
+  const freeink::book::BookStatus st = freeink::book::ImageRenderer::render(*paginator.bookSource(), paginator.zip(),
+                                                                            img, scratch, &G2Writer::onRow, &writer);
   writer.file.close();  // close before remove/rename below
   if (st != freeink::book::BookStatus::Ok || writer.failed) {
     LOG_ERR("FIBIMG", "Image decode failed (%d): %s", static_cast<int>(st), img.href);

src/activities/reader/KOReaderSyncActivity.cpp

@@ -86,11 +86,10 @@ void KOReaderSyncActivity::resolveRemotePosition() {
           ? std::clamp((remoteProgress.percentage - chapterStart) / (chapterEnd - chapterStart), 0.0f, 1.0f)
           : 0.0f;
 
-  const freeink::book::ManifestItem* item = paginator.book().spineItem(spine);
-  const freeink::book::ZipEntry* entry = item != nullptr ? paginator.book().zip().find(item->href) : nullptr;
-  if (entry == nullptr) return;
+  freeink::book::ZipEntry entry;
+  if (!paginator.spineZipEntry(spine, &entry)) return;
   uint32_t charStart = 0;
-  if (BookXPath::charStartForXpath(*paginator.bookSource(), paginator.book().zip(), *entry, remoteProgress.progress,
+  if (BookXPath::charStartForXpath(*paginator.bookSource(), paginator.zip(), entry, remoteProgress.progress,
                                    &charStart)) {
     remotePosition.charStart = charStart;
     remotePosition.hasCharStart = true;