Add SD catalog as default EPUB container backend

Changes the container backend strategy to use SD catalog by default for all EPUBs. On first open or stale index, builds the catalog once by borrowing the framebuffer (which can be released since e-ink holds the loading popup image). Subsequent opens use the fast path with compact resident tables. Falls back to legacy in-RAM open if catalog build fails due to IoError or OOM.

src/activities/reader/BookPaginator.cpp

@@ -76,14 +76,28 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G
   if (!isTxt_) {
     bool fbLent = false;
 
-    // Catalog fast path: an existing catalog.fibc means this container
+    // The SD catalog is the DEFAULT container backend for every EPUB.
-    // already outgrew the in-RAM probe ladder on a previous open -- load the
+    // Fast path: an existing catalog.fibc with a matching fingerprint opens
-    // compact resident tables and skip the ladder entirely. A stale index
+    // with no container parse at all -- just the compact resident tables
-    // (container changed) is removed inside openCatalog() and the ladder
+    // (a few KB for a normal book). A stale index (container changed) is
-    // below decides afresh what the book now needs.
+    // removed inside openCatalog() and rebuilt below.
     if (cache_.exists(freeink::book::BookCatalog::kCatalogName)) {
       catalogMode_ = openCatalog();
     }
+    if (!catalogMode_) {
+      // First open (or stale index): stream the container index to SD once,
+      // then every later open takes the fast path above. The build wants two
+      // ~50-96 KB arenas, so borrow the framebuffer -- the LOADING popup
+      // already on the panel survives (e-ink holds its image without the
+      // buffer).
+      renderer.releaseFrameBufferForBuild();
+      fbLent = true;
+      catalogMode_ = buildCatalog() && openCatalog();
+      if (!catalogMode_) {
+        LOG_INF("FIB", "Catalog build failed; falling back to in-RAM open");
+      }
+    }
+    // FALLBACK (catalog build failed: IoError, OOM): the legacy in-RAM open.
     // The whole book budget must coexist with a ~100+ KB chapter-build
     // scratch on a heap where the reader sees ~140 KB free, so worst-case
     // arena sizing is unaffordable. Open in two passes: parse into a
@@ -100,10 +114,9 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G
     // region the chapter builds need.
     size_t bookUsed = 0;
     if (!catalogMode_) {
-      // Probe ladder: most books fit the 48 KB rung; webnovel omnibuses
+      // Probe ladder: most books fit the 48 KB rung; larger containers need
-      // (1500+ spine items) need far more container metadata, so the bigger
+      // more metadata, so bigger rungs exist. The framebuffer is already
-      // rungs borrow the framebuffer (the LOADING popup already on the panel
+      // lent from the catalog-build attempt above.
-      // survives — e-ink holds its image without the buffer).
       static constexpr size_t kProbeSizes[] = {kBookArenaSize, 72 * 1024, 96 * 1024};
       BookStatus st = BookStatus::OutOfMemory;
       for (size_t attempt = 0; attempt < sizeof(kProbeSizes) / sizeof(kProbeSizes[0]); ++attempt) {
@@ -135,21 +148,9 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G
         if (st != BookStatus::OutOfMemory) break;  // real parse error, not arena size
         LOG_DBG("FIB", "Book probe outgrew %u B, retrying larger", static_cast<unsigned>(probeSize));
       }
-      if (st == BookStatus::OutOfMemory) {
+      // No further recourse when even the 96 KB rung overflows: the catalog
-        // Even the 96 KB rung overflowed: webnovel-omnibus territory (1,700+
+      // build (the path that handles 400 KB omnibus metadata) already failed
-        // spine items need ~400 KB of container metadata). Build the
+      // above, and its log says why.
-        // SD-backed catalog instead -- compact resident tables, everything
-        // string-shaped stays on the card. The framebuffer is already lent
-        // unless the heap itself was short, in which case the catalog build
-        // fails with its own clear log.
-        if (!fbLent) {
-          renderer.releaseFrameBufferForBuild();
-          fbLent = true;
-        }
-        LOG_INF("FIB", "Container outgrew the probe ladder; building SD catalog");
-        catalogMode_ = buildCatalog() && openCatalog();
-        if (catalogMode_) st = BookStatus::Ok;
-      }
       if (st != BookStatus::Ok) {
         LOG_ERR("FIB", "Book open failed: %d (%s)", static_cast<int>(st), path.c_str());
         if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart();
@@ -250,10 +251,15 @@ bool BookPaginator::openCatalog() {
 
 // Streams the container index to SD. Caller has the framebuffer lent; the
 // two arenas mirror the chapter-build split (record tables vs parse stream).
-// Measured for a 1,732-spine omnibus: records ~72.5 KB, parse ~45.3 KB.
+// The records demand scales with the entry count (~42 B each): measured
+// ~72.5 KB records + ~45.3 KB parse for a 1,732-spine omnibus, single-digit
+// KB records for a normal book. Take the ideal when the heap has it and
+// otherwise whatever the largest block gives down to a floor sized for
+// normal books; an omnibus that then overflows the arena fails the build
+// with a clear log and the caller falls back to the in-RAM ladder.
 bool BookPaginator::buildCatalog() {
   constexpr size_t kRecordsIdeal = 96 * 1024;
-  constexpr size_t kRecordsFloor = 76 * 1024;
+  constexpr size_t kRecordsFloor = 16 * 1024;
   constexpr size_t kParseSize = 50 * 1024;
   constexpr size_t kAllocSlack = 64;
 

src/activities/reader/BookPaginator.h

@@ -7,9 +7,13 @@
 //
 // Memory model (ESP32-C3, no PSRAM; the reader sees only ~140 KB free, so
 // nothing is sized for the worst case):
-//   book arena    exact-sized: open() parses into a temporary 48 KB arena to
+//   book arena    the SD-backed BookCatalog's resident tables, exact-sized
-//                 learn the real footprint (corpus high-water 5-40 KB), then
+//                 from the catalog footer (a few KB for a normal book,
-//                 re-opens into precisely that many bytes
+//                 ~42 KB for a 1,732-spine omnibus). Reopens skip the
+//                 container parse entirely. Only when the one-time catalog
+//                 build fails does open() fall back to the legacy in-RAM
+//                 Book: probe-parse into a temporary arena to learn the real
+//                 footprint, then re-open into precisely that many bytes
 //   stylesheet    compacted to the actual rule array (typically < 2 KB)
 //   index arena   12 KB  current chapter's page index + anchor table
 //   page arena    12 KB  decoded runs of the page being rendered
@@ -63,8 +67,9 @@ class BookPaginator {
   // Container lookups (image probes, internal links) -- the in-RAM catalog or
   // the SD-backed one, whichever this book opened with.
   const freeink::book::ZipCatalog& zip() const { return catalogMode_ ? catalog_.zip() : book_.zip(); }
-  // True when the book outgrew the in-RAM probe ladder and runs on the
+  // True when the book runs on the SD-backed BookCatalog -- the default for
-  // SD-backed BookCatalog (webnovel omnibuses).
+  // every EPUB; false only after a catalog build failure forced the legacy
+  // in-RAM fallback.
   bool isCatalogMode() const { return catalogMode_; }
   size_t spineCount() const {
     if (isTxt_) return 1;
@@ -192,10 +197,10 @@ class BookPaginator {
   uint32_t fontFingerprint() const;
   // Uncompressed bytes of a spine item -- the whole-book progress weights.
   uint32_t spineSizeAt(size_t spineIndex) const;
-  // SD-backed catalog path (omnibuses whose container outgrows the probe
+  // SD-backed catalog path (the default open path for every EPUB): open an
-  // ladder): open an existing catalog.fibc / build one. openCatalog removes
+  // existing catalog.fibc / build one. openCatalog removes a stale index
-  // a stale index (changed container) and returns false so the caller can
+  // (changed container) and returns false so the caller can rebuild;
-  // rebuild; buildCatalog expects the framebuffer already lent.
+  // buildCatalog expects the framebuffer already lent.
   bool openCatalog();
   bool buildCatalog();
   // Builds + compacts the book stylesheet (CSS items from whichever catalog).
@@ -215,8 +220,8 @@ class BookPaginator {
 
   SdBookSource source_;
   SdCacheStorage cache_;
-  freeink::book::Book book_;
+  freeink::book::Book book_;            // legacy in-RAM fallback (catalog build failed)
-  freeink::book::BookCatalog catalog_;  // SD-backed container index (omnibuses)
+  freeink::book::BookCatalog catalog_;  // SD-backed container index (default)
   freeink::book::LayoutParams params_;
   freeink::book::PageCacheReader reader_;
   freeink::book::FontChain chain_;