diff --git a/src/activities/reader/BookPaginator.cpp b/src/activities/reader/BookPaginator.cpp index 15d4448e..b435bab2 100644 --- a/src/activities/reader/BookPaginator.cpp +++ b/src/activities/reader/BookPaginator.cpp @@ -57,6 +57,7 @@ class ProgressSink : public freeink::book::PageSink { bool BookPaginator::open(const std::string& path, const std::string& cacheDir, GfxRenderer& renderer, const bool forcePlainText) { + cacheDir_ = cacheDir; close(); if (!source_.open(path.c_str())) { @@ -68,7 +69,21 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G const size_t len = path.size(); isTxt_ = forcePlainText || (len > 4 && strcasecmp(path.c_str() + len - 4, ".txt") == 0); + // The cache dir hosts both the page caches and (for omnibuses) the + // container index, so bind it before the catalog fast path below. + cache_.setDir(cacheDir.c_str()); + if (!isTxt_) { + bool fbLent = false; + + // Catalog fast path: an existing catalog.fibc means this container + // already outgrew the in-RAM probe ladder on a previous open -- load the + // compact resident tables and skip the ladder entirely. A stale index + // (container changed) is removed inside openCatalog() and the ladder + // below decides afresh what the book now needs. + if (cache_.exists(freeink::book::BookCatalog::kCatalogName)) { + catalogMode_ = openCatalog(); + } // The whole book budget must coexist with a ~100+ KB chapter-build // scratch on a heap where the reader sees ~140 KB free, so worst-case // arena sizing is unaffordable. Open in two passes: parse into a @@ -84,82 +99,107 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G // temp arena, both proved to pin large blocks mid-heap and split the // region the chapter builds need. size_t bookUsed = 0; - { + if (!catalogMode_) { + // Probe ladder: most books fit the 48 KB rung; webnovel omnibuses + // (1500+ spine items) need far more container metadata, so the bigger + // rungs borrow the framebuffer (the LOADING popup already on the panel + // survives — e-ink holds its image without the buffer). + static constexpr size_t kProbeSizes[] = {kBookArenaSize, 72 * 1024, 96 * 1024}; + BookStatus st = BookStatus::OutOfMemory; + for (size_t attempt = 0; attempt < sizeof(kProbeSizes) / sizeof(kProbeSizes[0]); ++attempt) { + const size_t probeSize = kProbeSizes[attempt]; + if (attempt > 0 && !fbLent) { + renderer.releaseFrameBufferForBuild(); + fbLent = true; + } + auto scratchBuf = makeUniqueNoThrow(kOpenScratchSize); + auto tempBookBuf = makeUniqueNoThrow(probeSize); + if (!scratchBuf || !tempBookBuf) { + LOG_ERR("FIB", "OOM: open working set (%u B, free heap %u)", + static_cast(kOpenScratchSize + probeSize), static_cast(ESP.getFreeHeap())); + st = BookStatus::OutOfMemory; + break; // the heap itself is short; a bigger rung cannot help + } + Arena scratch(scratchBuf.get(), kOpenScratchSize); + Arena tempArena(tempBookBuf.get(), probeSize); + freeink::book::Book probeBook; + st = probeBook.open(source_, tempArena, scratch); + if (st == BookStatus::Ok) { + bookUsed = tempArena.used(); + if (attempt > 0) { + LOG_INF("FIB", "Large container: book arena needs %u B (attempt %u)", static_cast(bookUsed), + static_cast(attempt + 1)); + } + break; + } + if (st != BookStatus::OutOfMemory) break; // real parse error, not arena size + LOG_DBG("FIB", "Book probe outgrew %u B, retrying larger", static_cast(probeSize)); + } + if (st == BookStatus::OutOfMemory) { + // Even the 96 KB rung overflowed: webnovel-omnibus territory (1,700+ + // spine items need ~400 KB of container metadata). Build the + // SD-backed catalog instead -- compact resident tables, everything + // string-shaped stays on the card. The framebuffer is already lent + // unless the heap itself was short, in which case the catalog build + // fails with its own clear log. + if (!fbLent) { + renderer.releaseFrameBufferForBuild(); + fbLent = true; + } + LOG_INF("FIB", "Container outgrew the probe ladder; building SD catalog"); + catalogMode_ = buildCatalog() && openCatalog(); + if (catalogMode_) st = BookStatus::Ok; + } + if (st != BookStatus::Ok) { + LOG_ERR("FIB", "Book open failed: %d (%s)", static_cast(st), path.c_str()); + if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart(); + close(); + return false; + } + } + + if (!catalogMode_) { + // PHASE B: exact-size arena into the emptied heap, re-parse for keeps. + bookBuf_ = makeUniqueNoThrow(bookUsed + 128); auto scratchBuf = makeUniqueNoThrow(kOpenScratchSize); - auto tempBookBuf = makeUniqueNoThrow(kBookArenaSize); - if (!scratchBuf || !tempBookBuf) { - LOG_ERR("FIB", "OOM: open working set (%u B, free heap %u)", - static_cast(kOpenScratchSize + kBookArenaSize), static_cast(ESP.getFreeHeap())); + if (!bookBuf_ || !scratchBuf) { + LOG_ERR("FIB", "OOM: book arena reopen (%u B)", static_cast(bookUsed + 128)); + if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart(); close(); return false; } Arena scratch(scratchBuf.get(), kOpenScratchSize); - Arena tempArena(tempBookBuf.get(), kBookArenaSize); - freeink::book::Book probeBook; - const BookStatus st = probeBook.open(source_, tempArena, scratch); + bookArena_.init(bookBuf_.get(), bookUsed + 128); + const BookStatus st = book_.open(source_, bookArena_, scratch); if (st != BookStatus::Ok) { - LOG_ERR("FIB", "Book open failed: %d (%s)", static_cast(st), path.c_str()); + LOG_ERR("FIB", "Exact-size reopen failed: %d", static_cast(st)); + if (fbLent && !renderer.restoreFrameBufferAfterBuild()) ESP.restart(); close(); return false; } - bookUsed = tempArena.used(); - } - - // PHASE B: exact-size arena into the emptied heap, re-parse for keeps. - bookBuf_ = makeUniqueNoThrow(bookUsed + 128); - auto scratchBuf = makeUniqueNoThrow(kOpenScratchSize); - if (!bookBuf_ || !scratchBuf) { - LOG_ERR("FIB", "OOM: book arena reopen (%u B)", static_cast(bookUsed + 128)); - close(); - return false; - } - Arena scratch(scratchBuf.get(), kOpenScratchSize); - bookArena_.init(bookBuf_.get(), bookUsed + 128); - const BookStatus st = book_.open(source_, bookArena_, scratch); - if (st != BookStatus::Ok) { - LOG_ERR("FIB", "Exact-size reopen failed: %d", static_cast(st)); - close(); - return false; - } - - // Stylesheet: build in a temporary working arena, keep only the rules. - auto tempSheetBuf = makeUniqueNoThrow(kSheetArenaSize); - if (tempSheetBuf) { - Arena sheetArena(tempSheetBuf.get(), kSheetArenaSize); - CssStylesheetBuilder builder; - if (builder.begin(sheetArena)) { - for (size_t m = 0; m < book_.manifestCount(); ++m) { - const ManifestItem* item = book_.manifestItem(m); - if (item == nullptr || item->mediaType == nullptr || strcmp(item->mediaType, "text/css") != 0) continue; - if (const freeink::book::ZipEntry* e = book_.zip().find(item->href)) { - builder.addSheet(source_, *e, scratch); - } - } - sheet_ = builder.finish(); - if (builder.skippedSheets() > 0) { - LOG_INF("FIB", "%u stylesheet(s) skipped (size cap)", builder.skippedSheets()); - } - if (sheet_.ruleCount > 0) { - // CssRule is self-contained (name hashes + POD declaration), so the - // rule array can move out of the 12 KB working arena wholesale. - const size_t rulesBytes = sheet_.ruleCount * sizeof(freeink::book::CssRule); - sheetBuf_ = makeUniqueNoThrow(rulesBytes); - if (sheetBuf_) { - memcpy(sheetBuf_.get(), sheet_.rules, rulesBytes); - sheet_.rules = reinterpret_cast(sheetBuf_.get()); - } else { - sheet_ = freeink::book::CssStylesheet{}; // OOM: lay out with element defaults - } - } + buildStylesheet(scratch); + scratchBuf.reset(); // free the reopen scratch before the framebuffer returns + } else if (catalog_.cssCount() > 0) { + // Catalog mode: CSS entries were resolved at catalog open; the build + // only needs a transient inflate-capable scratch. + auto scratchBuf = makeUniqueNoThrow(kOpenScratchSize); + if (scratchBuf) { + Arena scratch(scratchBuf.get(), kOpenScratchSize); + buildStylesheet(scratch); + } else { + LOG_ERR("FIB", "OOM: stylesheet scratch; using element defaults"); } } - LOG_INF("FIB", "Book open: %u spine items, book arena %u B (exact), %u CSS rules, free heap %u", - static_cast(book_.spineCount()), static_cast(bookUsed), sheet_.ruleCount, - static_cast(ESP.getFreeHeap())); - } + if (fbLent && !renderer.restoreFrameBufferAfterBuild()) { + LOG_ERR("FIB", "Framebuffer restore failed after large open"); + ESP.restart(); // transients freed; a failed 48 KB realloc means heap corruption + } - cache_.setDir(cacheDir.c_str()); + LOG_INF("FIB", "Book open%s: %u spine items, book arena %u B, %u CSS rules, free heap %u", + catalogMode_ ? " (SD catalog)" : "", static_cast(spineCount()), + static_cast(bookArena_.used()), sheet_.ruleCount, static_cast(ESP.getFreeHeap())); + } if (!buildFontChain(renderer)) { close(); @@ -174,10 +214,128 @@ bool BookPaginator::open(const std::string& path, const std::string& cacheDir, G return true; } +// Loads an existing catalog.fibc into an exactly-sized resident arena. A +// stale index (container changed since the build) is deleted so the caller +// can rebuild; any other failure just reports false. +bool BookPaginator::openCatalog() { + size_t resident = 0; + BookStatus st = freeink::book::BookCatalog::residentBytes(cache_, &resident); + if (st != BookStatus::Ok) return false; + + bookBuf_ = makeUniqueNoThrow(resident); + auto scratchBuf = makeUniqueNoThrow(4096); // fingerprint scan buffer + if (!bookBuf_ || !scratchBuf) { + LOG_ERR("FIB", "OOM: catalog resident arena (%u B)", static_cast(resident)); + bookBuf_.reset(); + return false; + } + bookArena_.init(bookBuf_.get(), resident); + Arena scratch(scratchBuf.get(), 4096); + st = catalog_.open(source_, cache_, bookArena_, scratch); + if (st == BookStatus::Stale) { + LOG_INF("FIB", "Catalog stale (container changed); rebuilding"); + cache_.remove(freeink::book::BookCatalog::kCatalogName); + bookBuf_.reset(); + return false; + } + if (st != BookStatus::Ok) { + LOG_ERR("FIB", "Catalog open failed: %d", static_cast(st)); + bookBuf_.reset(); + return false; + } + LOG_INF("FIB", "SD catalog open: %u spines, %u toc, resident %u B", static_cast(catalog_.spineCount()), + static_cast(catalog_.tocCount()), static_cast(bookArena_.used())); + return true; +} + +// Streams the container index to SD. Caller has the framebuffer lent; the +// two arenas mirror the chapter-build split (record tables vs parse stream). +// Measured for a 1,732-spine omnibus: records ~72.5 KB, parse ~45.3 KB. +bool BookPaginator::buildCatalog() { + constexpr size_t kRecordsIdeal = 96 * 1024; + constexpr size_t kRecordsFloor = 76 * 1024; + constexpr size_t kParseSize = 50 * 1024; + constexpr size_t kAllocSlack = 64; + + auto parseBuf = makeUniqueNoThrow(kParseSize); + if (!parseBuf) { + LOG_ERR("FIB", "OOM: catalog parse arena (free heap %u)", static_cast(ESP.getFreeHeap())); + return false; + } + const size_t block = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); + const size_t recordsSize = std::min(kRecordsIdeal, block > kAllocSlack ? block - kAllocSlack : 0); + std::unique_ptr recordsBuf; + if (recordsSize >= kRecordsFloor) recordsBuf = makeUniqueNoThrow(recordsSize); + if (!recordsBuf) { + LOG_ERR("FIB", "OOM: catalog record arena (want %u B, max block %u)", static_cast(recordsSize), + static_cast(block)); + return false; + } + Arena records(recordsBuf.get(), recordsSize); + Arena parse(parseBuf.get(), kParseSize); + + const uint32_t t0 = millis(); + const BookStatus st = freeink::book::BookCatalog::build(source_, cache_, records, &parse); + if (st != BookStatus::Ok) { + LOG_ERR("FIB", "Catalog build failed: %d (records %u/%u refused %u, parse %u/%u refused %u B)", + static_cast(st), static_cast(records.highWater()), static_cast(recordsSize), + static_cast(records.failedAllocSize()), static_cast(parse.highWater()), + static_cast(kParseSize), static_cast(parse.failedAllocSize())); + return false; + } + LOG_INF("FIB", "Catalog built in %ums (records %u/%u, parse %u/%u B)", static_cast(millis() - t0), + static_cast(records.highWater()), static_cast(recordsSize), + static_cast(parse.highWater()), static_cast(kParseSize)); + return true; +} + +// Builds the book stylesheet in a temporary working arena and keeps only the +// compacted rule array. CSS items come from whichever catalog is active. +void BookPaginator::buildStylesheet(Arena& scratch) { + auto tempSheetBuf = makeUniqueNoThrow(kSheetArenaSize); + if (!tempSheetBuf) return; + Arena sheetArena(tempSheetBuf.get(), kSheetArenaSize); + CssStylesheetBuilder builder; + if (!builder.begin(sheetArena)) return; + if (catalogMode_) { + for (size_t c = 0; c < catalog_.cssCount(); ++c) { + builder.addSheet(source_, *catalog_.cssEntry(c), scratch); + } + } else { + for (size_t m = 0; m < book_.manifestCount(); ++m) { + const ManifestItem* item = book_.manifestItem(m); + if (item == nullptr || item->mediaType == nullptr || strcmp(item->mediaType, "text/css") != 0) continue; + if (const freeink::book::ZipEntry* e = book_.zip().find(item->href)) { + builder.addSheet(source_, *e, scratch); + } + } + } + sheet_ = builder.finish(); + if (builder.skippedSheets() > 0) { + LOG_INF("FIB", "%u stylesheet(s) skipped (size cap)", builder.skippedSheets()); + } + if (sheet_.ruleCount > 0) { + // CssRule is self-contained (name hashes + POD declaration), so the + // rule array can move out of the 12 KB working arena wholesale. + const size_t rulesBytes = sheet_.ruleCount * sizeof(freeink::book::CssRule); + sheetBuf_ = makeUniqueNoThrow(rulesBytes); + if (sheetBuf_) { + memcpy(sheetBuf_.get(), sheet_.rules, rulesBytes); + sheet_.rules = reinterpret_cast(sheetBuf_.get()); + } else { + sheet_ = freeink::book::CssStylesheet{}; // OOM: lay out with element defaults + } + } +} + void BookPaginator::close() { + suspendBuild(); // commits a partial so the next open resumes instantly + chapterSource_.close(); source_.close(); reader_ = freeink::book::PageCacheReader(); book_ = freeink::book::Book(); + catalog_ = freeink::book::BookCatalog(); + catalogMode_ = false; sheet_ = freeink::book::CssStylesheet{}; chain_ = freeink::book::FontChain(); hyphBlob_.reset(); @@ -192,9 +350,8 @@ void BookPaginator::close() { } const char* BookPaginator::language() const { - if (!isTxt_ && book_.metadata().language != nullptr && book_.metadata().language[0] != '\0') { - return book_.metadata().language; - } + const char* lang = catalogMode_ ? catalog_.metadata().language : book_.metadata().language; + if (!isTxt_ && lang != nullptr && lang[0] != '\0') return lang; return "en"; } @@ -340,26 +497,82 @@ bool BookPaginator::isChapterCached(const uint16_t spineIndex) { return cache_.exists(name); } -freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex, const BuildProgress& progress) { +freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex, const BuildProgress& progress, + const uint32_t targetChar, const float targetFraction) { const uint32_t gen = generation(); if (!freeink::book::pageCacheName(spineIndex, gen, cacheName_, sizeof(cacheName_))) { return BookStatus::Unsupported; } + // An in-progress incremental build of this same chapter/generation stays + // live across ensureChapter calls (page turns re-enter here). + if (building_) { + if (buildGeneration_ == gen && curSpine_ == spineIndex) return BookStatus::Ok; + suspendBuild(); // settings/spine changed under the build + } + curSpine_ = kNoSpine; + partialReaderOpen_ = false; if (!indexBuf_ && !reallocChapterArenas()) return BookStatus::OutOfMemory; indexArena_.reset(); BookStatus st = reader_.open(cache_, cacheName_, gen, indexArena_); - if (st == BookStatus::Ok) { + if (st == BookStatus::Ok && !reader_.isPartial()) { curSpine_ = spineIndex; return st; } - const ManifestItem* item = isTxt_ ? nullptr : book_.spineItem(spineIndex); - const freeink::book::ZipEntry* entry = (!isTxt_ && item != nullptr) ? book_.zip().find(item->href) : nullptr; - if (!isTxt_ && entry == nullptr) return BookStatus::NotFound; + // Resolve the chapter's entry + href into MEMBERS: in catalog mode the + // ZipEntry is materialized from the SD index (no arena-resident copy + // exists), and the incremental session retains pointers across steps. + const freeink::book::ZipEntry* entry = nullptr; + if (!isTxt_) { + if (catalogMode_) { + if (catalog_.spineEntry(spineIndex, &curEntry_) != BookStatus::Ok || + catalog_.spineHref(spineIndex, chapterHref_, sizeof(chapterHref_)) != BookStatus::Ok) { + return BookStatus::NotFound; + } + } else { + const ManifestItem* item = book_.spineItem(spineIndex); + const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr; + if (e == nullptr) return BookStatus::NotFound; + curEntry_ = *e; + snprintf(chapterHref_, sizeof(chapterHref_), "%s", item->href); + } + entry = &curEntry_; + } if (isTxt_ && spineIndex != 0) return BookStatus::NotFound; + // A suspended partial serves its pages instantly while a fresh background + // rebuild re-lays the chapter from the start (deterministic layout makes + // the rebuilt prefix identical). The partial file stays readable during + // the rebuild because writes stream to a temp name until commit. + const bool incrementalLanding = targetChar != kBuildAll || targetFraction >= 0.0f; + if (st == BookStatus::Ok && reader_.isPartial() && !isTxt_ && incrementalLanding) { + partialReaderOpen_ = true; + const BookStatus inc = startIncremental(spineIndex, *entry, targetChar, targetFraction, progress); + if (inc == BookStatus::Ok) { + curSpine_ = spineIndex; + return BookStatus::Ok; + } + // Incremental could not start (extraction/OOM): fall through to the + // blocking build below, which replaces the partial outright. + partialReaderOpen_ = false; + } + + // Giant uncached chapter with a known landing position: build only a + // little past it and finish behind the reader. + constexpr uint32_t kIncrementalThreshold = 96 * 1024; // uncompressed bytes + if (!isTxt_ && incrementalLanding && entry->uncompressedSize > kIncrementalThreshold) { + reader_ = freeink::book::PageCacheReader(); + indexBuf_.reset(); // nothing to serve from it; reclaim 12 KB for the session + const BookStatus inc = startIncremental(spineIndex, *entry, targetChar, targetFraction, progress); + if (inc == BookStatus::Ok) { + curSpine_ = spineIndex; + return BookStatus::Ok; + } + // fall through to the blocking build + } + // The pagination working set lives only for this call, as TWO arenas: // layout buffers + the writer's index in one, the parse stream (inflate // window + decompressor + XML chunks) in the other. Splitting kills the @@ -380,11 +593,10 @@ freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex // reserve is being squeezed — raise it first. constexpr size_t kParserHeapReserve = 22 * 1024; constexpr size_t kLayoutIdeal = 60 * 1024; - // Floor is measured-demand-plus-margin, not comfort: heaviest observed - // chapter (device, incl. chunked writer index) is 44,964 B. Raising this - // makes builds fail upfront on heap shapes where they would have fit — - // a 46 KiB floor already rejected a 46,452 B offer that would have built. - constexpr size_t kLayoutFloor = 45 * 1024; + // Floor is measured-demand-plus-margin, not comfort. The LayoutEngine + // itself now lives in this arena (session refactor moved it off the + // stack): small-profile host high water incl. writer index is 48,656 B. + constexpr size_t kLayoutFloor = 48 * 1024; constexpr size_t kParseIdeal = 50 * 1024; constexpr size_t kParseFloor = 46 * 1024; constexpr size_t kParseStored = 8 * 1024; // stored entries skip inflate state @@ -456,7 +668,7 @@ freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex ProgressSink sink(writer, progress); uint32_t totalChars = 0; st = isTxt_ ? ChapterLayout::layoutPlainText(source_, params_, scratch, sink, nullptr, &totalChars) - : ChapterLayout::layout(source_, book_.zip(), *entry, item->href, params_, scratch, sink, nullptr, + : ChapterLayout::layout(source_, zip(), *entry, chapterHref_, params_, scratch, sink, nullptr, &totalChars, &parseArena); writer.setTotalChars(totalChars); if (st == BookStatus::Ok && !writer.finish()) st = BookStatus::IoError; @@ -485,9 +697,264 @@ freeink::book::BookStatus BookPaginator::ensureChapter(const uint16_t spineIndex return st; } +// Extracts (inflates) a deflated chapter to /xNNNN.raw once, so the +// layout session parses a stored file (~8 KB resident parse state instead of +// the ~46 KB inflate stream). The file is keyed on the spine index only — it +// survives settings changes (generations) and is reused by rebuilds. +bool BookPaginator::extractChapter(const uint16_t spineIndex, const freeink::book::ZipEntry& entry) { + char rawName[16]; + snprintf(rawName, sizeof(rawName), "x%04u.raw", spineIndex); + char rawPath[128]; + snprintf(rawPath, sizeof(rawPath), "%s/%s", cacheDir_.c_str(), rawName); + + chapterSource_.close(); + if (!Storage.exists(rawPath) || (chapterSource_.open(rawPath) && chapterSource_.size() != entry.uncompressedSize)) { + chapterSource_.close(); + // Inflate the whole entry to SD. Transient: one inflate stream (~46 KB). + auto scratchBuf = makeUniqueNoThrow(kOpenScratchSize); + if (!scratchBuf) { + LOG_ERR("FIB", "OOM: chapter extract scratch"); + return false; + } + Arena scratch(scratchBuf.get(), kOpenScratchSize); + freeink::book::ZipEntryReader zr; + if (zr.open(source_, entry, scratch) != BookStatus::Ok) return false; + uint8_t* buf = static_cast(scratch.alloc(4096, 1)); + if (buf == nullptr) return false; + char tmpPath[136]; + snprintf(tmpPath, sizeof(tmpPath), "%s.tmp", rawPath); + { + HalFile out; + if (!Storage.openFileForWrite("FIB", tmpPath, out)) return false; + for (;;) { + const int32_t n = zr.read(buf, 4096); + if (n < 0) { + Storage.remove(tmpPath); + return false; + } + if (n == 0) break; + if (out.write(buf, n) != static_cast(n)) { + Storage.remove(tmpPath); + return false; + } + } + } + Storage.remove(rawPath); + if (!Storage.rename(tmpPath, rawPath)) return false; + if (!chapterSource_.open(rawPath)) return false; + } + rawEntry_ = freeink::book::ZipEntryReader::rawEntry(entry.uncompressedSize); + return true; +} + +freeink::book::BookStatus BookPaginator::startIncremental(const uint16_t spineIndex, + const freeink::book::ZipEntry& entry, + const uint32_t targetChar, const float targetFraction, + const BuildProgress& progress) { + // Resident while reading: layout (buffers + engine + writer index chunks — + // a ~700-page chapter needs ~6 chunks) + a SMALL parse arena (the chapter + // is stored/extracted, so no inflate window). The image pre-scan's probe + // inflate (~46 KB) runs in a TRANSIENT arena freed right after begin() — + // keeping it out of the resident set is what lets the session coexist + // with the framebuffer and font caches on the C3. Expat pools stay on the + // system heap (~19 KB while the session lives). + constexpr size_t kBuildLayoutSize = 56 * 1024; + constexpr size_t kBuildParseSize = 12 * 1024; + constexpr size_t kPrescanSize = 46 * 1024; + constexpr uint32_t kWindowPages = 2; // build this far past the landing page + + // ensureChapter resolved curEntry_/chapterHref_ before calling here; the + // `entry` argument aliases curEntry_ (a member -- the session's readers + // retain pointers into it across steps). + freeink::book::BookSource* chapterSrc = &source_; + const freeink::book::ZipEntry* parseEntry = &entry; + if (entry.method != 0) { + if (!extractChapter(spineIndex, entry)) return BookStatus::IoError; + chapterSrc = &chapterSource_; + parseEntry = &rawEntry_; + } + + buildLayoutBuf_ = makeUniqueNoThrow(kBuildLayoutSize); + buildParseBuf_ = makeUniqueNoThrow(kBuildParseSize); + if (!buildLayoutBuf_ || !buildParseBuf_) { + LOG_ERR("FIB", "OOM: incremental build arenas (free heap %u)", static_cast(ESP.getFreeHeap())); + buildLayoutBuf_.reset(); + buildParseBuf_.reset(); + return BookStatus::OutOfMemory; + } + buildLayoutArena_.init(buildLayoutBuf_.get(), kBuildLayoutSize); + buildParseArena_.init(buildParseBuf_.get(), kBuildParseSize); + + buildWriter_ = freeink::book::PageCacheWriter(); + const uint32_t gen = generation(); + if (!buildWriter_.begin(cache_, cacheName_, gen, buildLayoutArena_)) { + buildLayoutBuf_.reset(); + buildParseBuf_.reset(); + return BookStatus::IoError; + } + BookStatus st; + { + // Transient probe arena for begin() only (image dimension pre-scan). + auto prescanBuf = makeUniqueNoThrow(kPrescanSize); + Arena prescanArena(prescanBuf.get(), prescanBuf ? kPrescanSize : 0); + st = buildSession_.begin(source_, &zip(), *chapterSrc, *parseEntry, chapterHref_, params_, buildLayoutArena_, + buildWriter_, &buildParseArena_, prescanBuf ? &prescanArena : nullptr); + } + if (st != BookStatus::Ok) { + LOG_ERR("FIB", "Incremental begin failed: %d", static_cast(st)); + buildSession_.abort(); + buildWriter_ = freeink::book::PageCacheWriter(); + buildLayoutBuf_.reset(); + buildParseBuf_.reset(); + return st; + } + buildGeneration_ = gen; + building_ = true; + + // Synchronous burst: lay out just past the landing position. For a char + // target, "the target's page + window exists" is the stop (pageForChar is + // a watermark). For a fraction target the stop is the parsed BYTE ratio — + // the caller then resolves its landing char against chars-so-far, which by + // construction sits right at this stop point. + const uint32_t t0 = millis(); + while (!buildSession_.done()) { + if (targetFraction >= 0.0f) { + const uint64_t total = buildSession_.bytesTotal(); + if (total > 0 && + buildSession_.bytesConsumed() >= static_cast(targetFraction * static_cast(total)) && + buildWriter_.pageCount() > kWindowPages) { + break; + } + } else if (buildWriter_.pageCount() > buildWriter_.pageForChar(targetChar) + kWindowPages) { + break; + } + st = buildSession_.step(4); + if (st != BookStatus::Ok) { + LOG_ERR("FIB", "Incremental build failed: %d (layout %u/%u refused %u, parse %u/%u refused %u B)", + static_cast(st), static_cast(buildLayoutArena_.highWater()), + static_cast(kBuildLayoutSize), static_cast(buildLayoutArena_.failedAllocSize()), + static_cast(buildParseArena_.highWater()), static_cast(kBuildParseSize), + static_cast(buildParseArena_.failedAllocSize())); + suspendBuild(); + return st; + } + if (progress.fn != nullptr) progress.fn(progress.ctx, buildWriter_.pageCount()); + } + if (buildSession_.done()) return finalizeBuild(); + LOG_INF("FIB", "Chapter %u building incrementally: %u pages to target in %ums (of ~%u est)", spineIndex, + buildWriter_.pageCount(), static_cast(millis() - t0), static_cast(estimatedTotalPages())); + return BookStatus::Ok; +} + +freeink::book::BookStatus BookPaginator::pumpBuild(const uint32_t pages) { + if (!building_) return BookStatus::Ok; + const BookStatus st = buildSession_.step(pages); + if (st != BookStatus::Ok) { + LOG_ERR("FIB", "Incremental pump failed: %d", static_cast(st)); + suspendBuild(); + return st; + } + if (buildSession_.done()) return finalizeBuild(); + return BookStatus::Ok; +} + +freeink::book::BookStatus BookPaginator::finalizeBuild() { + buildWriter_.setTotalChars(buildSession_.totalChars()); + const bool committed = buildWriter_.finish(); + const uint32_t pages = buildWriter_.pageCount(); + buildSession_.abort(); + buildWriter_ = freeink::book::PageCacheWriter(); + buildLayoutBuf_.reset(); + buildParseBuf_.reset(); + building_ = false; + partialReaderOpen_ = false; + + reader_ = freeink::book::PageCacheReader(); + if (!indexBuf_ && !reallocChapterArenas()) return BookStatus::OutOfMemory; + indexArena_.reset(); + if (!committed) return BookStatus::IoError; + const BookStatus st = reader_.open(cache_, cacheName_, buildGeneration_, indexArena_); + LOG_INF("FIB", "Incremental build finalized: %u pages (open %d, free heap %u)", pages, static_cast(st), + static_cast(ESP.getFreeHeap())); + return st; +} + +void BookPaginator::suspendBuild() { + if (!building_) return; + buildWriter_.setTotalChars(buildSession_.totalChars()); // chars-so-far watermark + buildWriter_.suspend(static_cast(buildSession_.bytesConsumed()), + static_cast(buildSession_.bytesTotal())); + buildSession_.abort(); + buildWriter_ = freeink::book::PageCacheWriter(); + buildLayoutBuf_.reset(); + buildParseBuf_.reset(); + building_ = false; + partialReaderOpen_ = false; + curSpine_ = kNoSpine; // force a clean reopen (of the partial) next time +} + +uint32_t BookPaginator::pageCount() const { + if (!building_) return reader_.pageCount(); + const uint32_t w = buildWriter_.pageCount(); + const uint32_t r = partialReaderOpen_ ? reader_.pageCount() : 0; + return w > r ? w : r; +} + +uint32_t BookPaginator::totalChars() const { + if (!building_) return reader_.totalChars(); + // A chars-so-far watermark: the denominator grows as the build advances, + // exactly like the legacy incremental build. Display code uses + // estimatedTotalPages() instead. + const uint32_t w = buildSession_.totalChars(); + const uint32_t r = partialReaderOpen_ ? reader_.totalChars() : 0; + return w > r ? w : r; +} + +uint32_t BookPaginator::charStartOfPage(const uint32_t pageIndex) const { + if (!building_) return reader_.charStart(pageIndex); + if (pageIndex < buildWriter_.pageCount()) return buildWriter_.charStart(pageIndex); + return partialReaderOpen_ ? reader_.charStart(pageIndex) : 0; +} + +uint32_t BookPaginator::pageForChar(const uint32_t charOffset) const { + if (!building_) return reader_.pageForChar(charOffset); + // The rebuilt prefix is byte-identical to the partial's, so the indexes + // agree wherever they overlap — just ask the source that has more pages. + if (partialReaderOpen_ && reader_.pageCount() > buildWriter_.pageCount()) { + return reader_.pageForChar(charOffset); + } + return buildWriter_.pageForChar(charOffset); +} + +uint32_t BookPaginator::estimatedTotalPages() const { + const uint32_t known = pageCount(); + uint64_t consumed = 0; + uint64_t total = 0; + uint64_t basePages = 0; + if (building_) { + consumed = buildSession_.bytesConsumed(); + total = buildSession_.bytesTotal(); + basePages = buildWriter_.pageCount(); + } else if (reader_.isPartial()) { + consumed = reader_.buildBytesConsumed(); + total = reader_.buildBytesTotal(); + basePages = reader_.pageCount(); + } else { + return known; + } + if (consumed == 0 || total == 0 || basePages == 0) return known; + const uint64_t est = basePages * total / consumed; + return est > known ? static_cast(est) : known; +} + bool BookPaginator::charForAnchor(const char* fragment, uint32_t* charOut) const { if (fragment == nullptr || fragment[0] == '\0') return false; - return reader_.charForAnchor(ZipCatalog::hashPath(fragment), charOut); + const uint32_t hash = ZipCatalog::hashPath(fragment); + if (building_) { + if (buildWriter_.charForAnchor(hash, charOut)) return true; + return partialReaderOpen_ && reader_.charForAnchor(hash, charOut); + } + return reader_.charForAnchor(hash, charOut); } freeink::book::BookStatus BookPaginator::readPage(const uint32_t pageIndex, freeink::book::Page* out) { @@ -495,11 +962,19 @@ freeink::book::BookStatus BookPaginator::readPage(const uint32_t pageIndex, free // a failed reallocation (the arena would point at freed memory). if (curSpine_ == kNoSpine || !pageBuf_) return BookStatus::NotFound; pageArena_.reset(); + if (building_) { + if (pageIndex < buildWriter_.pageCount()) return buildWriter_.readPage(pageIndex, pageArena_, out); + if (partialReaderOpen_ && pageIndex < reader_.pageCount()) { + return reader_.readPage(pageIndex, pageArena_, out); + } + return BookStatus::NotFound; // beyond the watermark — pump and retry + } return reader_.readPage(pageIndex, pageArena_, out); } int BookPaginator::spineIndexForHref(const char* href) const { if (href == nullptr || href[0] == '\0' || isTxt_) return -1; + if (catalogMode_) return catalog_.spineIndexForHref(href); for (size_t s = 0; s < book_.spineCount(); ++s) { const ManifestItem* item = book_.spineItem(s); if (item != nullptr && strcmp(item->href, href) == 0) return static_cast(s); @@ -507,9 +982,36 @@ int BookPaginator::spineIndexForHref(const char* href) const { return -1; } +bool BookPaginator::spineZipEntry(const int spineIndex, freeink::book::ZipEntry* out) const { + if (isTxt_ || spineIndex < 0) return false; + if (catalogMode_) { + return catalog_.spineEntry(static_cast(spineIndex), out) == BookStatus::Ok; + } + const ManifestItem* item = book_.spineItem(static_cast(spineIndex)); + const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr; + if (e == nullptr) return false; + *out = *e; + return true; +} + BookPaginator::TocItem BookPaginator::tocItem(const size_t index) const { TocItem out{"", nullptr, -1, 0}; - const freeink::book::TocEntry* entry = isTxt_ ? nullptr : book_.tocEntry(index); + if (isTxt_) return out; + if (catalogMode_) { + // Title/fragment are SD reads into the shared member buffers -- valid + // until the next tocItem() call (see the TocItem declaration). + freeink::book::BookCatalog::TocItem item; + if (catalog_.tocItem(index, &item, tocTitleBuf_, sizeof(tocTitleBuf_), tocFragBuf_, sizeof(tocFragBuf_)) != + BookStatus::Ok) { + return out; + } + out.title = tocTitleBuf_; + out.fragment = item.hasFragment ? tocFragBuf_ : nullptr; + out.spineIndex = item.spineIndex; + out.depth = item.depth; + return out; + } + const freeink::book::TocEntry* entry = book_.tocEntry(index); if (entry == nullptr) return out; out.title = entry->title; out.fragment = entry->fragment; @@ -521,6 +1023,7 @@ BookPaginator::TocItem BookPaginator::tocItem(const size_t index) const { int BookPaginator::tocIndexForSpine(const int spineIndex) const { // The chapter's title is the last TOC entry at or before this spine item // (a spine item without its own entry belongs to the preceding heading). + if (catalogMode_) return catalog_.tocIndexForSpine(spineIndex); int best = -1; int bestSpine = -1; for (size_t t = 0; t < tocCount(); ++t) { @@ -534,7 +1037,14 @@ int BookPaginator::tocIndexForSpine(const int spineIndex) const { return best; } -// Spine weights use the uncompressed sizes already in the ZIP catalog — the +uint32_t BookPaginator::spineSizeAt(const size_t spineIndex) const { + if (catalogMode_) return catalog_.spineSize(spineIndex); + const ManifestItem* item = book_.spineItem(spineIndex); + const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr; + return e != nullptr ? e->uncompressedSize : 0; +} + +// Spine weights use the uncompressed sizes already in the catalog -- the // same "bigger chapters cover more of the book" heuristic the legacy engine // used, with zero extra state. float BookPaginator::bookProgress(const int spineIndex, const float chapterFraction) const { @@ -542,10 +1052,8 @@ float BookPaginator::bookProgress(const int spineIndex, const float chapterFract uint64_t before = 0; uint64_t current = 0; uint64_t total = 0; - for (size_t s = 0; s < book_.spineCount(); ++s) { - const ManifestItem* item = book_.spineItem(s); - const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr; - const uint32_t size = e != nullptr ? e->uncompressedSize : 0; + for (size_t s = 0; s < spineCount(); ++s) { + const uint32_t size = spineSizeAt(s); if (static_cast(s) < spineIndex) before += size; if (static_cast(s) == spineIndex) current = size; total += size; @@ -557,24 +1065,20 @@ float BookPaginator::bookProgress(const int spineIndex, const float chapterFract int BookPaginator::spineForBookFraction(const float bookFraction, float* chapterFractionOut) const { if (chapterFractionOut != nullptr) *chapterFractionOut = 0.0f; - if (isTxt_ || book_.spineCount() == 0) { + if (isTxt_ || spineCount() == 0) { if (chapterFractionOut != nullptr) *chapterFractionOut = bookFraction; return 0; } uint64_t total = 0; - for (size_t s = 0; s < book_.spineCount(); ++s) { - const ManifestItem* item = book_.spineItem(s); - const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr; - total += e != nullptr ? e->uncompressedSize : 0; + for (size_t s = 0; s < spineCount(); ++s) { + total += spineSizeAt(s); } const float f = bookFraction < 0.0f ? 0.0f : (bookFraction > 1.0f ? 1.0f : bookFraction); const uint64_t target = static_cast(f * static_cast(total)); uint64_t cumulative = 0; - for (size_t s = 0; s < book_.spineCount(); ++s) { - const ManifestItem* item = book_.spineItem(s); - const freeink::book::ZipEntry* e = item != nullptr ? book_.zip().find(item->href) : nullptr; - const uint32_t size = e != nullptr ? e->uncompressedSize : 0; - if (target < cumulative + size || s + 1 == book_.spineCount()) { + for (size_t s = 0; s < spineCount(); ++s) { + const uint32_t size = spineSizeAt(s); + if (target < cumulative + size || s + 1 == spineCount()) { if (chapterFractionOut != nullptr && size > 0) { *chapterFractionOut = static_cast(target - cumulative) / static_cast(size); } diff --git a/src/activities/reader/BookPaginator.h b/src/activities/reader/BookPaginator.h index c4994024..66ab720b 100644 --- a/src/activities/reader/BookPaginator.h +++ b/src/activities/reader/BookPaginator.h @@ -20,6 +20,7 @@ // produce one contiguous block // Steady-state page turns touch only the page arena (~2 KB used). +#include #include #include #include @@ -58,21 +59,40 @@ class BookPaginator { bool isOpen() const { return open_; } bool isTxt() const { return isTxt_; } - freeink::book::Book& book() { return book_; } freeink::book::BookSource* bookSource() { return &source_; } - size_t spineCount() const { return isTxt_ ? 1 : book_.spineCount(); } + // Container lookups (image probes, internal links) -- the in-RAM catalog or + // the SD-backed one, whichever this book opened with. + const freeink::book::ZipCatalog& zip() const { return catalogMode_ ? catalog_.zip() : book_.zip(); } + // True when the book outgrew the in-RAM probe ladder and runs on the + // SD-backed BookCatalog (webnovel omnibuses). + bool isCatalogMode() const { return catalogMode_; } + size_t spineCount() const { + if (isTxt_) return 1; + return catalogMode_ ? catalog_.spineCount() : book_.spineCount(); + } const char* language() const; - const char* title() const { return isTxt_ ? "" : book_.metadata().title; } - const char* author() const { return isTxt_ ? "" : book_.metadata().author; } + const char* title() const { + return isTxt_ ? "" : (catalogMode_ ? catalog_.metadata().title : book_.metadata().title); + } + const char* author() const { + return isTxt_ ? "" : (catalogMode_ ? catalog_.metadata().author : book_.metadata().author); + } + // ZipEntry of a spine item (KOSync/xpath bridges). False when absent. + bool spineZipEntry(int spineIndex, freeink::book::ZipEntry* out) const; // --- TOC (flattened, resolved to spine indices) -------------------------- struct TocItem { + // In catalog mode both strings live in a shared internal buffer that the + // NEXT tocItem() call overwrites -- use or copy them before iterating on. const char* title; const char* fragment; // anchor within the chapter, or nullptr int spineIndex; // -1 when the href is not a spine item uint8_t depth; }; - size_t tocCount() const { return isTxt_ ? 0 : book_.tocCount(); } + size_t tocCount() const { + if (isTxt_) return 0; + return catalogMode_ ? catalog_.tocCount() : book_.tocCount(); + } TocItem tocItem(size_t index) const; // First TOC entry pointing at `spineIndex` or an earlier chapter (the // chapter's display title); -1 when the TOC has no such entry. @@ -100,14 +120,41 @@ class BookPaginator { // Opens the chapter's page cache for the current generation, laying the // chapter out first when missing or stale. Heavy only on that miss. - freeink::book::BookStatus ensureChapter(uint16_t spineIndex, const BuildProgress& progress = BuildProgress()); + // + // `targetChar` — the landing position (chapter character offset), when the + // caller knows it. For a GIANT uncached chapter (or a suspended partial) + // this switches to the INCREMENTAL path: the chapter builds only a little + // past the target, isBuilding() turns true, pages serve from the partial + // build, and the caller finishes the rest via pumpBuild() between page + // turns. kBuildAll (the default) forces the classic blocking full build — + // required for percent jumps (they need the final page count). + // `targetFraction` (0..1) is the fraction-of-chapter alternative for + // landings that predate exact charStart positions (migrated legacy + // progress, whole-book percent jumps): the incremental build runs until + // the parsed BYTE ratio passes it — bytes track extracted characters + // closely enough that the subsequent pageForChar lands within a page. + static constexpr uint32_t kBuildAll = 0xFFFFFFFF; + freeink::book::BookStatus ensureChapter(uint16_t spineIndex, const BuildProgress& progress = BuildProgress(), + uint32_t targetChar = kBuildAll, float targetFraction = -1.0f); bool chapterReady() const { return curSpine_ != kNoSpine; } uint16_t currentSpine() const { return curSpine_; } - uint32_t pageCount() const { return reader_.pageCount(); } - uint32_t totalChars() const { return reader_.totalChars(); } - uint32_t charStartOfPage(uint32_t pageIndex) const { return reader_.charStart(pageIndex); } - uint32_t pageForChar(uint32_t charOffset) const { return reader_.pageForChar(charOffset); } + // --- incremental build (giant single-spine chapters) --------------------- + bool isBuilding() const { return building_; } + // Lays out up to `pages` more pages of the in-progress build; finalizes + // (writer commit + cache reopen) when the chapter ends. Callers gate this + // on a read-ahead window (currentPage + N) — never pump unboundedly. + freeink::book::BookStatus pumpBuild(uint32_t pages); + // Byte-ratio estimate of the final page count while building ("page X of + // ~Y"); the exact count once the build (or cache) is complete. + uint32_t estimatedTotalPages() const; + + // While building incrementally these merge the live writer (rebuild + // watermark) with any reopened partial — see the .cpp implementations. + uint32_t pageCount() const; + uint32_t totalChars() const; + uint32_t charStartOfPage(uint32_t pageIndex) const; + uint32_t pageForChar(uint32_t charOffset) const; // Resolve an id="" fragment in the CURRENT chapter to a char offset. bool charForAnchor(const char* fragment, uint32_t* charOut) const; @@ -143,10 +190,33 @@ class BookPaginator { bool reallocChapterArenas(); void loadHyphenator(); uint32_t fontFingerprint() const; + // Uncompressed bytes of a spine item -- the whole-book progress weights. + uint32_t spineSizeAt(size_t spineIndex) const; + // SD-backed catalog path (omnibuses whose container outgrows the probe + // ladder): open an existing catalog.fibc / build one. openCatalog removes + // a stale index (changed container) and returns false so the caller can + // rebuild; buildCatalog expects the framebuffer already lent. + bool openCatalog(); + bool buildCatalog(); + // Builds + compacts the book stylesheet (CSS items from whichever catalog). + void buildStylesheet(freeink::book::Arena& scratch); + + // Incremental-build internals. A "giant" chapter (uncompressed size above + // kIncrementalThreshold) is extracted once to /xNNNN.raw so the + // resident parse state is the stored-entry ~8 KB plus probe headroom, then + // laid out through a ChapterLayoutSession feeding buildWriter_. Session + // arenas (buildLayoutBuf_/buildParseBuf_) stay allocated while reading; + // suspendBuild() commits a partial cache on exit/spine-switch. + freeink::book::BookStatus startIncremental(uint16_t spineIndex, const freeink::book::ZipEntry& entry, + uint32_t targetChar, float targetFraction, const BuildProgress& progress); + freeink::book::BookStatus finalizeBuild(); + void suspendBuild(); + bool extractChapter(uint16_t spineIndex, const freeink::book::ZipEntry& entry); SdBookSource source_; SdCacheStorage cache_; freeink::book::Book book_; + freeink::book::BookCatalog catalog_; // SD-backed container index (omnibuses) freeink::book::LayoutParams params_; freeink::book::PageCacheReader reader_; freeink::book::FontChain chain_; @@ -176,4 +246,28 @@ class BookPaginator { uint16_t curSpine_ = kNoSpine; bool open_ = false; bool isTxt_ = false; + bool catalogMode_ = false; + std::string cacheDir_; // per-book cache directory (extraction files live here) + + // Current chapter identity. curEntry_ must be a MEMBER: the incremental + // layout session's ZipEntryReader retains a pointer to it across steps, and + // in catalog mode there is no arena-resident entry to point at. + freeink::book::ZipEntry curEntry_{}; + char chapterHref_[512] = ""; + // tocItem() string backing in catalog mode (single slot, see TocItem note). + mutable char tocTitleBuf_[256]; + mutable char tocFragBuf_[256]; + + // Incremental-build session state (live only while building_). + freeink::book::ChapterLayoutSession buildSession_; + freeink::book::PageCacheWriter buildWriter_; + SdBookSource chapterSource_; // the extracted raw chapter file + freeink::book::ZipEntry rawEntry_{}; // headerless entry over chapterSource_ + std::unique_ptr buildLayoutBuf_; // session layout arena backing + std::unique_ptr buildParseBuf_; // session parse arena backing + freeink::book::Arena buildLayoutArena_; + freeink::book::Arena buildParseArena_; + uint32_t buildGeneration_ = 0; // generation the session was started under + bool building_ = false; + bool partialReaderOpen_ = false; // reader_ holds a partial while rebuilding }; diff --git a/src/activities/reader/EpubReaderActivity.cpp b/src/activities/reader/EpubReaderActivity.cpp index 89580d13..d378f505 100644 --- a/src/activities/reader/EpubReaderActivity.cpp +++ b/src/activities/reader/EpubReaderActivity.cpp @@ -202,7 +202,7 @@ float EpubReaderActivity::currentBookFraction() const { void EpubReaderActivity::openReaderMenu() { const int currentPageDisplay = paginator.chapterReady() ? static_cast(currentPage) + 1 : 0; - const int totalPages = paginator.chapterReady() ? static_cast(paginator.pageCount()) : 0; + const int totalPages = paginator.chapterReady() ? static_cast(paginator.estimatedTotalPages()) : 0; const int bookProgressPercent = clampPercent(static_cast(currentBookFraction() * 100.0f + 0.5f)); startActivityForResult( std::make_unique(renderer, mappedInput, paginator.title(), currentPageDisplay, totalPages, @@ -251,6 +251,19 @@ void EpubReaderActivity::loop() { pendingReadFolderMove = false; } + // Incremental chapter build: keep a small window laid out ahead of the + // reader, in short bursts, only when no render is in flight. NEVER pump + // unboundedly — an ungated background build once locked the legacy reader + // solid. A chapter bigger than the window simply stays partial (suspended + // to disk on exit) and follows the reader; a small remainder finalizes. + if (paginator.isBuilding() && !RenderLock::peek()) { + constexpr uint32_t kWindowAhead = 5; + if (paginator.pageCount() < static_cast(currentPage) + 1 + kWindowAhead) { + RenderLock lock; + if (paginator.isBuilding()) paginator.pumpBuild(2); + } + } + if (automaticPageTurnActive) { if (mappedInput.wasReleased(MappedInputManager::Button::Confirm) || mappedInput.wasReleased(MappedInputManager::Button::Back)) { @@ -562,11 +575,10 @@ bool EpubReaderActivity::launchKOReaderSync() { // the whole-book percentage is the robust fallback. std::string localXPath = syntheticXPath(currentSpineIndex); if (!paginator.isTxt()) { - const freeink::book::ManifestItem* item = paginator.book().spineItem(currentSpineIndex); - const freeink::book::ZipEntry* entry = item != nullptr ? paginator.book().zip().find(item->href) : nullptr; - if (entry != nullptr) { - localXPath = BookXPath::xpathForCharStart(*paginator.bookSource(), paginator.book().zip(), *entry, - currentSpineIndex, lastCharStart); + freeink::book::ZipEntry entry; + if (paginator.spineZipEntry(currentSpineIndex, &entry)) { + localXPath = BookXPath::xpathForCharStart(*paginator.bookSource(), paginator.zip(), entry, currentSpineIndex, + lastCharStart); } } SavedProgressPosition localKoPos{std::move(localXPath), currentBookFraction()}; @@ -625,7 +637,9 @@ void EpubReaderActivity::toggleAutoPageTurn(const uint8_t selectedPageTurnOption void EpubReaderActivity::pageTurn(bool isForwardTurn) { if (isForwardTurn) { - if (paginator.chapterReady() && currentPage + 1 < paginator.pageCount()) { + // While a chapter is still building, pageCount() is a watermark, not the + // end of the chapter — keep advancing; render() builds the page in. + if (paginator.chapterReady() && (currentPage + 1 < paginator.pageCount() || paginator.isBuilding())) { currentPage++; } else { currentSpineIndex++; // spineCount == end-of-book screen @@ -691,7 +705,23 @@ bool EpubReaderActivity::ensureChapterAndPosition() { } }; - const auto status = paginator.ensureChapter(static_cast(currentSpineIndex), progressCb); + // Landing target: a known charStart (resume, reanchor, footnote return), + // a chapter fraction (migrated legacy progress, percent jumps), or the + // chapter top (plain page turns) all go INCREMENTAL on giant chapters — + // the build runs just past the landing point and finishes behind the + // reader. Only anchors (need the full anchor map) and last-page entries + // (need the final count) take the classic blocking build (kBuildAll). + uint32_t targetChar = BookPaginator::kBuildAll; + float targetFraction = -1.0f; + if (pendingCharStart.has_value()) { + targetChar = *pendingCharStart; + } else if (pendingChapterFraction.has_value()) { + targetFraction = *pendingChapterFraction; + } else if (pendingAnchor.empty() && !pendingLastPage) { + targetChar = 0; // entering at the chapter top + } + const auto status = + paginator.ensureChapter(static_cast(currentSpineIndex), progressCb, targetChar, targetFraction); if (!renderer.hasFrameBuffer() && !renderer.restoreFrameBufferAfterBuild()) { // Unrecoverable: nothing can be drawn again. The build arenas are freed // (ensureChapter released them), so this realloc failing means the heap @@ -727,9 +757,22 @@ bool EpubReaderActivity::ensureChapterAndPosition() { pendingChapterFraction.reset(); pendingLastPage = false; } else if (pendingChapterFraction.has_value()) { - const uint32_t targetChar = - static_cast(*pendingChapterFraction * static_cast(paginator.totalChars())); - currentPage = paginator.pageForChar(targetChar); + if (paginator.isBuilding()) { + // totalChars() is a build watermark here: fraction x watermark would + // land at fraction-of-the-PREFIX (half the intended depth for a 50% + // resume). Resolve against the estimated total page count instead; + // the build stopped right at the fraction's byte offset, so the + // clamped watermark page IS the landing page. + const uint32_t est = paginator.estimatedTotalPages(); + const uint32_t built = paginator.pageCount(); + uint32_t page = static_cast(*pendingChapterFraction * static_cast(est)); + if (page >= built) page = built > 0 ? built - 1 : 0; + currentPage = page; + } else { + const uint32_t targetChar = + static_cast(*pendingChapterFraction * static_cast(paginator.totalChars())); + currentPage = paginator.pageForChar(targetChar); + } pendingChapterFraction.reset(); pendingLastPage = false; } else if (pendingLastPage) { @@ -737,8 +780,22 @@ bool EpubReaderActivity::ensureChapterAndPosition() { pendingLastPage = false; } + // Reaching past the last built page of an in-progress build means "keep + // building" (handled in render), not "clamp": pageCount() is a watermark. + if (paginator.isBuilding() && currentPage >= paginator.pageCount()) { + constexpr uint32_t kCatchUpBurst = 4; + if (!buildPopupShown && currentPage >= paginator.pageCount() + 2) { + // A deep jump has real work ahead of it; the burst below blocks. + GUI.drawPopup(renderer, tr(STR_INDEXING)); + pagesUntilFullRefresh = 1; + buildPopupShown = true; + } + while (paginator.isBuilding() && currentPage >= paginator.pageCount()) { + if (paginator.pumpBuild(kCatchUpBurst) != freeink::book::BookStatus::Ok) break; + } + } if (paginator.pageCount() > 0 && currentPage >= paginator.pageCount()) { - currentPage = paginator.pageCount() - 1; + currentPage = paginator.pageCount() - 1; // true end of chapter (or build failure) } return true; } @@ -959,7 +1016,10 @@ void EpubReaderActivity::renderPage(const freeink::book::Page& page, int) { void EpubReaderActivity::renderStatusBar() const { const int currentPageDisplay = static_cast(currentPage) + 1; - const float pageCount = paginator.chapterReady() ? static_cast(paginator.pageCount()) : 0.0f; + // estimatedTotalPages() equals pageCount() once the chapter is fully + // built; while building it is the byte-ratio estimate, so "page X of Y" + // does not display the watermark as if it were the chapter total. + const float pageCount = paginator.chapterReady() ? static_cast(paginator.estimatedTotalPages()) : 0.0f; const float bookProgress = currentBookFraction() * 100.0f; std::string title; diff --git a/src/activities/reader/FreeInkBookStorage.h b/src/activities/reader/FreeInkBookStorage.h index d2c29e17..94438d95 100644 --- a/src/activities/reader/FreeInkBookStorage.h +++ b/src/activities/reader/FreeInkBookStorage.h @@ -91,6 +91,23 @@ class SdCacheStorage : public freeink::book::CacheStorage { return true; } + // Mid-build page read-back (incremental layout): the write handle is + // opened O_RDWR by SDCardManager, so seek-read-seek-back on the SAME + // handle is coherent — the exact pattern the legacy engine used. The + // write cursor MUST be restored or subsequent writes corrupt the cache; + // if that seek fails, fail loudly. + int32_t readBackAt(uint32_t offset, void* dst, uint32_t len) override { + if (!write_.isOpen()) return -1; + const size_t writePos = write_.position(); + if (!write_.seekSet(offset)) return -1; + const int32_t n = write_.read(dst, len); + if (!write_.seekSet(writePos)) { + LOG_ERR("FIBCACHE", "readBackAt: write cursor restore FAILED"); + return -1; + } + return n; + } + private: static constexpr const char* kTempName = "_tmp.fibp"; diff --git a/src/activities/reader/FreeInkPageRenderer.cpp b/src/activities/reader/FreeInkPageRenderer.cpp index ce4544b4..80751736 100644 --- a/src/activities/reader/FreeInkPageRenderer.cpp +++ b/src/activities/reader/FreeInkPageRenderer.cpp @@ -100,8 +100,8 @@ bool ensureImageCached(BookPaginator& paginator, const std::string& cacheDir, co static_cast(img.height & 0xFF), static_cast(img.height >> 8)}; writer.failed = writer.file.write(header, sizeof(header)) != sizeof(header); - const freeink::book::BookStatus st = freeink::book::ImageRenderer::render( - *paginator.bookSource(), paginator.book().zip(), img, scratch, &G2Writer::onRow, &writer); + const freeink::book::BookStatus st = freeink::book::ImageRenderer::render(*paginator.bookSource(), paginator.zip(), + img, scratch, &G2Writer::onRow, &writer); writer.file.close(); // close before remove/rename below if (st != freeink::book::BookStatus::Ok || writer.failed) { LOG_ERR("FIBIMG", "Image decode failed (%d): %s", static_cast(st), img.href); diff --git a/src/activities/reader/KOReaderSyncActivity.cpp b/src/activities/reader/KOReaderSyncActivity.cpp index 605896dd..892deeaf 100644 --- a/src/activities/reader/KOReaderSyncActivity.cpp +++ b/src/activities/reader/KOReaderSyncActivity.cpp @@ -86,11 +86,10 @@ void KOReaderSyncActivity::resolveRemotePosition() { ? std::clamp((remoteProgress.percentage - chapterStart) / (chapterEnd - chapterStart), 0.0f, 1.0f) : 0.0f; - const freeink::book::ManifestItem* item = paginator.book().spineItem(spine); - const freeink::book::ZipEntry* entry = item != nullptr ? paginator.book().zip().find(item->href) : nullptr; - if (entry == nullptr) return; + freeink::book::ZipEntry entry; + if (!paginator.spineZipEntry(spine, &entry)) return; uint32_t charStart = 0; - if (BookXPath::charStartForXpath(*paginator.bookSource(), paginator.book().zip(), *entry, remoteProgress.progress, + if (BookXPath::charStartForXpath(*paginator.bookSource(), paginator.zip(), entry, remoteProgress.progress, &charStart)) { remotePosition.charStart = charStart; remotePosition.hasCharStart = true;