From 9d147cade93ad6aa391b1b7515b8b19b612f56f8 Mon Sep 17 00:00:00 2001 From: Justin Mitchell Date: Mon, 20 Jul 2026 17:16:54 -0400 Subject: [PATCH] Deduplicate identical CSS files in EPUB parsing & probe images for dimensions instead of reading the full file Some EPUB converters emit byte-identical stylesheets per chapter (100+ entries). Now scan the ZIP central directory once to identify duplicates by CRC32 and compressed size, then skip parsing identical files. This avoids redundant ZIP lookups and SD extraction round-trips while preserving all styles since rules merge into a global set. Also add extractItemToFile helper and allowEarlyStop parameter to readItemContentsToStream. --- lib/Epub/Epub.cpp | 60 +++++++- lib/Epub/Epub.h | 5 +- lib/Epub/Epub/Section.cpp | 5 +- lib/Epub/Epub/blocks/ImageBlock.cpp | 27 +++- lib/Epub/Epub/blocks/ImageBlock.h | 14 +- lib/Epub/Epub/converters/ImageDimsProbe.cpp | 144 ++++++++++++++++++ lib/Epub/Epub/converters/ImageDimsProbe.h | 49 ++++++ .../Epub/parsers/ChapterHtmlSlimParser.cpp | 63 +++++--- lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h | 1 + lib/ZipFile/ZipFile.cpp | 11 +- lib/ZipFile/ZipFile.h | 21 ++- src/activities/reader/EpubReaderActivity.cpp | 60 ++++++-- src/activities/reader/EpubReaderActivity.h | 10 ++ 13 files changed, 420 insertions(+), 50 deletions(-) create mode 100644 lib/Epub/Epub/converters/ImageDimsProbe.cpp create mode 100644 lib/Epub/Epub/converters/ImageDimsProbe.h diff --git a/lib/Epub/Epub.cpp b/lib/Epub/Epub.cpp index 13b4885e..e2958c84 100644 --- a/lib/Epub/Epub.cpp +++ b/lib/Epub/Epub.cpp @@ -256,8 +256,44 @@ void Epub::parseCssFiles() const { return; } + // Some converters emit one byte-identical stylesheet per chapter (100+ .css + // entries), and each parse costs a zip locate plus an SD extract round-trip. + // Map every CSS path to its central-directory (CRC32, compressed size) in a + // single scan and parse only the first of each identical pair. Rules merge + // into one global set, so dropping exact duplicates cannot lose styles. A + // path that never matches a directory entry keeps key 0 and always parses. + std::vector dedupKeys(cssFiles.size(), 0); + if (cssFiles.size() > 1) { + std::unordered_map pathToIndex; + pathToIndex.reserve(cssFiles.size()); + for (size_t i = 0; i < cssFiles.size(); i++) { + pathToIndex.emplace(FsHelpers::normalisePath(cssFiles[i]), i); + } + ZipFile(filepath).enumerateFileEntries([&](std::string_view entryPath, uint32_t crc32, uint32_t compressedSize) { + if (!FsHelpers::hasCssExtension(entryPath)) { + return; + } + const auto it = pathToIndex.find(std::string{entryPath}); + if (it != pathToIndex.end()) { + dedupKeys[it->second] = (static_cast(crc32) << 32) | compressedSize; + } + }); + } + std::vector seenKeys; + seenKeys.reserve(cssFiles.size()); + size_t skippedDuplicates = 0; + // No cache yet - parse CSS files - for (const auto& cssPath : cssFiles) { + for (size_t cssIndex = 0; cssIndex < cssFiles.size(); cssIndex++) { + const auto& cssPath = cssFiles[cssIndex]; + const uint64_t dedupKey = dedupKeys[cssIndex]; + if (dedupKey != 0) { + if (std::find(seenKeys.begin(), seenKeys.end(), dedupKey) != seenKeys.end()) { + skippedDuplicates++; + continue; + } + seenKeys.push_back(dedupKey); + } LOG_DBG("EBP", "Parsing CSS file: %s", cssPath.c_str()); // Check heap before parsing - CSS parsing allocates heavily @@ -312,7 +348,8 @@ void Epub::parseCssFiles() const { LOG_ERR("EBP", "Failed to save CSS rules to cache"); } - LOG_DBG("EBP", "Loaded %zu CSS style rules from %zu files", cssParser->ruleCount(), cssFiles.size()); + LOG_DBG("EBP", "Loaded %zu CSS style rules from %zu files (%zu identical duplicates skipped)", + cssParser->ruleCount(), cssFiles.size(), skippedDuplicates); cssParser->clear(); } @@ -728,14 +765,29 @@ uint8_t* Epub::readItemContentsToBytes(const std::string& itemHref, size_t* size return content; } -bool Epub::readItemContentsToStream(const std::string& itemHref, Print& out, const size_t chunkSize) const { +bool Epub::readItemContentsToStream(const std::string& itemHref, Print& out, const size_t chunkSize, + const bool allowEarlyStop) const { if (itemHref.empty()) { LOG_DBG("EBP", "Failed to read item, empty href"); return false; } const std::string path = FsHelpers::normalisePath(itemHref); - return ZipFile(filepath).readFileToStream(path.c_str(), out, chunkSize); + return ZipFile(filepath).readFileToStream(path.c_str(), out, chunkSize, allowEarlyStop); +} + +bool Epub::extractItemToFile(const std::string& itemHref, const std::string& destPath) const { + HalFile out; + if (!Storage.openFileForWrite("EBP", destPath, out)) { + return false; + } + const bool ok = readItemContentsToStream(itemHref, out, 4096); + out.flush(); + out.close(); + if (!ok) { + Storage.remove(destPath.c_str()); + } + return ok; } bool Epub::getItemSize(const std::string& itemHref, size_t* size) const { diff --git a/lib/Epub/Epub.h b/lib/Epub/Epub.h index 3c8ed39b..40ab6f25 100644 --- a/lib/Epub/Epub.h +++ b/lib/Epub/Epub.h @@ -59,7 +59,10 @@ class Epub { bool generateThumbBmp(int height) const; uint8_t* readItemContentsToBytes(const std::string& itemHref, size_t* size = nullptr, bool trailingNullByte = false) const; - bool readItemContentsToStream(const std::string& itemHref, Print& out, size_t chunkSize) const; + bool readItemContentsToStream(const std::string& itemHref, Print& out, size_t chunkSize, + bool allowEarlyStop = false) const; + // Extract an item to a file on SD. On failure the partial file is removed. + bool extractItemToFile(const std::string& itemHref, const std::string& destPath) const; bool getItemSize(const std::string& itemHref, size_t* size) const; BookMetadataCache::SpineEntry getSpineItem(int spineIndex) const; BookMetadataCache::TocEntry getTocItem(int tocIndex) const; diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index 10d2b4f3..2a311f0b 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -17,7 +17,10 @@ namespace { // v30: Arabic shaping changed both drawing and measurement (getTextAdvanceX now // measures the shaped visual text); cached word positions from v29 no longer // match what drawText renders. -constexpr uint8_t SECTION_FILE_VERSION = 31; +// v32: ImageBlock serializes the book-internal source href after the cache path +// (lazy extraction: images are header-probed at build time and extracted on +// first render). +constexpr uint8_t SECTION_FILE_VERSION = 32; // Written into the version field while a build is in progress; patched to // SECTION_FILE_VERSION only when the build is finalized. An abandoned / // crash-interrupted .bin therefore carries version 0, which loadSectionFile rejects diff --git a/lib/Epub/Epub/blocks/ImageBlock.cpp b/lib/Epub/Epub/blocks/ImageBlock.cpp index aa99951b..7209d999 100644 --- a/lib/Epub/Epub/blocks/ImageBlock.cpp +++ b/lib/Epub/Epub/blocks/ImageBlock.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "Epub/converters/DirectPixelWriter.h" #include "Epub/converters/ImageDecoderFactory.h" @@ -15,8 +16,16 @@ // - uint16_t height // - uint8_t pixels[...] - 2 bits per pixel, packed (4 pixels per byte), row-major order -ImageBlock::ImageBlock(const std::string& imagePath, int16_t width, int16_t height) - : imagePath(imagePath), width(width), height(height) {} +ImageBlock::ImageBlock(const std::string& imagePath, const std::string& srcPath, int16_t width, int16_t height) + : imagePath(imagePath), srcPath(srcPath), width(width), height(height) {} + +void* ImageBlock::extractCtx = nullptr; +ImageBlock::ExtractFn ImageBlock::extractFn = nullptr; + +void ImageBlock::setExtractor(void* ctx, ExtractFn fn) { + extractCtx = ctx; + extractFn = fn; +} bool ImageBlock::imageExists() const { return Storage.exists(imagePath.c_str()); } @@ -225,6 +234,15 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { return; // Successfully rendered from cache } + // The build only header-probed the image for dimensions; pull the actual + // file out of the book now, on first visit to the page. + if (!srcPath.empty() && extractFn && !Storage.exists(imagePath.c_str())) { + LOG_DBG("IMG", "Lazy-extracting %s -> %s", srcPath.c_str(), imagePath.c_str()); + if (!extractFn(extractCtx, srcPath.c_str(), imagePath.c_str())) { + LOG_ERR("IMG", "Lazy extraction failed: %s", srcPath.c_str()); + } + } + // No cache - need to decode the image // Check if image file exists HalFile file; @@ -280,6 +298,7 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { bool ImageBlock::serialize(HalFile& file) { serialization::writeString(file, imagePath); + serialization::writeString(file, srcPath); serialization::writePod(file, width); serialization::writePod(file, height); return true; @@ -287,9 +306,11 @@ bool ImageBlock::serialize(HalFile& file) { std::unique_ptr ImageBlock::deserialize(HalFile& file) { std::string path; + std::string src; serialization::readString(file, path); + serialization::readString(file, src); int16_t w, h; serialization::readPod(file, w); serialization::readPod(file, h); - return std::unique_ptr(new ImageBlock(path, w, h)); + return std::unique_ptr(new (std::nothrow) ImageBlock(path, src, w, h)); } diff --git a/lib/Epub/Epub/blocks/ImageBlock.h b/lib/Epub/Epub/blocks/ImageBlock.h index 886b5e98..cf6b59d3 100644 --- a/lib/Epub/Epub/blocks/ImageBlock.h +++ b/lib/Epub/Epub/blocks/ImageBlock.h @@ -8,7 +8,7 @@ class ImageBlock final : public Block { public: - ImageBlock(const std::string& imagePath, int16_t width, int16_t height); + ImageBlock(const std::string& imagePath, const std::string& srcPath, int16_t width, int16_t height); ~ImageBlock() override = default; const std::string& getImagePath() const { return imagePath; } @@ -21,6 +21,14 @@ class ImageBlock final : public Block { void renderPlaceholder(GfxRenderer& renderer, int x, int y) const; static void clearSessionRenderFailures(); + // Lazy extraction hook: the section build only header-probes images for their + // dimensions; the file at imagePath is extracted out of the book on first + // render, via this callback (function pointer + context, not std::function — + // this is render-loop code). Registered by the reader activity that owns the + // Epub, cleared on its exit. + using ExtractFn = bool (*)(void* ctx, const char* srcPath, const char* destPath); + static void setExtractor(void* ctx, ExtractFn fn); + BlockType getType() override { return IMAGE_BLOCK; } bool isEmpty() override { return false; } @@ -30,6 +38,10 @@ class ImageBlock final : public Block { private: std::string imagePath; + std::string srcPath; // book-internal source href; empty once known-extracted int16_t width; int16_t height; + + static void* extractCtx; + static ExtractFn extractFn; }; diff --git a/lib/Epub/Epub/converters/ImageDimsProbe.cpp b/lib/Epub/Epub/converters/ImageDimsProbe.cpp new file mode 100644 index 00000000..50faa97e --- /dev/null +++ b/lib/Epub/Epub/converters/ImageDimsProbe.cpp @@ -0,0 +1,144 @@ +#include "ImageDimsProbe.h" + +#include + +namespace { +// SOFn markers carry the frame dimensions. C4 (DHT), C8 (JPG extension) and +// CC (DAC) share the 0xCn range but are not frame headers. +bool isJpegSof(const uint8_t marker) { + return marker >= 0xC0 && marker <= 0xCF && marker != 0xC4 && marker != 0xC8 && marker != 0xCC; +} +constexpr uint8_t PNG_SIG[8] = {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}; +} // namespace + +bool ImageDimsProbe::feed(const uint8_t b) { + switch (state) { + case State::Sniff: + if (b == 0xFF) { + state = State::JpegSoi; + } else if (b == PNG_SIG[0]) { + state = State::PngHeader; + } else { + state = State::Failed; + return false; + } + pos = 1; + return true; + + case State::PngHeader: + // Bytes 1..7: signature; 8..11: IHDR length; 12..15: "IHDR"; 16..23: dims. + if (pos < 8) { + if (b != PNG_SIG[pos]) { + state = State::Failed; + return false; + } + } else if (pos >= 12 && pos < 16) { + if (b != "IHDR"[pos - 12]) { + state = State::Failed; + return false; + } + } else if (pos >= 16 && pos < 20) { + width = static_cast((static_cast(width) << 8) | b); + } else if (pos >= 20 && pos < 24) { + height = static_cast((static_cast(height) << 8) | b); + if (pos == 23) { + state = State::Done; + pos++; + return false; + } + } + pos++; + return true; + + case State::JpegSoi: + if (b != 0xD8) { + state = State::Failed; + return false; + } + state = State::JpegFf; + return true; + + case State::JpegFf: + if (b != 0xFF) { + state = State::Failed; + return false; + } + state = State::JpegMarker; + return true; + + case State::JpegMarker: + if (b == 0xFF) return true; // fill bytes before a marker are legal + if (b == 0x01 || (b >= 0xD0 && b <= 0xD8)) { + // TEM / RSTn / SOI: standalone, no length field. + state = State::JpegFf; + return true; + } + if (b == 0xD9 || b == 0xDA) { + // EOI or SOS before any SOF: no dimensions to be found. + state = State::Failed; + return false; + } + sofPending = isJpegSof(b); + state = State::JpegLenHi; + return true; + + case State::JpegLenHi: + segLen = static_cast(b << 8); + state = State::JpegLenLo; + return true; + + case State::JpegLenLo: + segLen = static_cast(segLen | b); + if (segLen < 2 || (sofPending && segLen < 7)) { + state = State::Failed; + return false; + } + if (sofPending) { + sofFill = 0; + state = State::JpegSof; + } else if (segLen == 2) { + state = State::JpegFf; + } else { + skipLeft = static_cast(segLen) - 2; + state = State::JpegSkip; + } + return true; + + case State::JpegSkip: + if (--skipLeft == 0) state = State::JpegFf; + return true; + + case State::JpegSof: + sofBuf[sofFill++] = b; + if (sofFill == 5) { + height = static_cast((sofBuf[1] << 8) | sofBuf[2]); + width = static_cast((sofBuf[3] << 8) | sofBuf[4]); + state = State::Done; + return false; + } + return true; + + case State::Done: + case State::Failed: + return false; + } + return false; +} + +size_t ImageDimsProbe::write(const uint8_t b) { return feed(b) ? 1 : 0; } + +size_t ImageDimsProbe::write(const uint8_t* data, const size_t len) { + for (size_t i = 0; i < len; i++) { + if (!feed(data[i])) return i; // short write: polite early stop + } + return len; +} + +bool ImageDimsProbe::getDimensions(ImageDimensions& out) const { + if (state != State::Done || width == 0 || height == 0 || width > INT16_MAX || height > INT16_MAX) { + return false; + } + out.width = static_cast(width); + out.height = static_cast(height); + return true; +} diff --git a/lib/Epub/Epub/converters/ImageDimsProbe.h b/lib/Epub/Epub/converters/ImageDimsProbe.h new file mode 100644 index 00000000..622d3004 --- /dev/null +++ b/lib/Epub/Epub/converters/ImageDimsProbe.h @@ -0,0 +1,49 @@ +#pragma once +#include + +#include "ImageToFramebufferDecoder.h" + +// Streaming JPEG/PNG header parser: finds image dimensions from the first few +// KB of a compressed stream without inflating the whole image. Feed bytes via +// the Print interface (e.g. Epub::readItemContentsToStream with +// allowEarlyStop=true); write() returns short once the dimensions are known or +// the stream is known to be unusable, which the zip layer treats as a polite +// early stop rather than an error. +// +// JPEG: walks marker segments (skipping EXIF/APPn of any size statefully, so +// nothing is buffered) until a SOFn frame header yields the dimensions. +// PNG: reads the IHDR fields at their fixed offsets (bytes 16..23). +class ImageDimsProbe : public Print { + public: + size_t write(uint8_t b) override; + size_t write(const uint8_t* data, size_t len) override; + + // True only when a valid header was found; fills `out`. + bool getDimensions(ImageDimensions& out) const; + + private: + bool feed(uint8_t b); // returns false once parsing is finished (found or failed) + + enum class State : uint8_t { + Sniff, // first byte decides the format + PngHeader, // PNG signature + IHDR at fixed offsets + JpegSoi, // second SOI byte (0xD8) + JpegFf, // expect a 0xFF marker prefix + JpegMarker, // marker type byte (0xFF padding allowed) + JpegLenHi, // segment length, high byte + JpegLenLo, // segment length, low byte + JpegSkip, // skipping a non-SOF segment body + JpegSof, // collecting the 5 SOF bytes: precision, height(2), width(2) + Done, + Failed, + }; + State state = State::Sniff; + uint32_t pos = 0; // absolute stream offset (PNG fixed-offset parsing) + uint32_t skipLeft = 0; // remaining segment bytes to skip + uint16_t segLen = 0; + bool sofPending = false; // current segment is a SOF frame header + uint8_t sofBuf[5] = {0}; + uint8_t sofFill = 0; + uint16_t width = 0; + uint16_t height = 0; +}; diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index a686b366..69dedeaa 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -15,6 +15,7 @@ #include "Epub.h" #include "Epub/Page.h" #include "Epub/converters/ImageDecoderFactory.h" +#include "Epub/converters/ImageDimsProbe.h" #include "Epub/converters/ImageToFramebufferDecoder.h" #include "Epub/htmlEntities.h" @@ -554,28 +555,47 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } std::string cachedImagePath = self->imageBasePath + std::to_string(self->imageCounter++) + ext; - // Extract image to cache file - HalFile cachedImageFile; - bool extractSuccess = false; - if (Storage.openFileForWrite("EHP", cachedImagePath, cachedImageFile)) { - extractSuccess = self->epub->readItemContentsToStream(resolvedPath, cachedImageFile, 4096); - cachedImageFile.flush(); - cachedImageFile.close(); - } - - if (extractSuccess) { - // Get image dimensions, retrying to absorb SD-card sync latency on slow - // cards. Replaces a blanket delay(50) that cost ~50ms on every image, and - // closes the silent-drop bug where a single getDimensions failure was fatal. + { + // Probe the dimensions from the entry's first bytes (early-aborted + // inflate, a few KB) instead of extracting the whole image now — + // extraction is deferred to the first render of the page (see + // ImageBlock's lazy extractor). This is what keeps first-open of an + // image-heavy chapter from stalling for seconds per image. ImageDimensions dims = {0, 0}; - ImageToFramebufferDecoder* decoder = ImageDecoderFactory::getDecoder(cachedImagePath); - bool gotDimensions = false; - for (int attempt = 0; attempt < 3 && !gotDimensions; attempt++) { - if (attempt > 0) { - delay(50); // Give a slow SD card time to finish syncing before retrying + ImageDimsProbe headerProbe; + self->epub->readItemContentsToStream(resolvedPath, headerProbe, 1024, /*allowEarlyStop=*/true); + bool gotDimensions = headerProbe.getDimensions(dims); + + if (!gotDimensions) { + // No header within the stream (rare) — fall back to extracting the + // whole image and probing the file. That can take seconds, so + // surface the indexing popup first (single-shot per parser). + if (self->popupFn && !self->imagePopupFired) { + self->imagePopupFired = true; + self->popupFn(); + } + HalFile cachedImageFile; + bool extractSuccess = false; + if (Storage.openFileForWrite("EHP", cachedImagePath, cachedImageFile)) { + extractSuccess = self->epub->readItemContentsToStream(resolvedPath, cachedImageFile, 4096); + cachedImageFile.flush(); + cachedImageFile.close(); + } + if (extractSuccess) { + // Retry to absorb SD-card sync latency on slow cards, and to close + // the silent-drop bug where a single getDimensions failure was fatal. + ImageToFramebufferDecoder* decoder = ImageDecoderFactory::getDecoder(cachedImagePath); + for (int attempt = 0; attempt < 3 && !gotDimensions; attempt++) { + if (attempt > 0) { + delay(50); // Give a slow SD card time to finish syncing before retrying + } + gotDimensions = decoder && decoder->getDimensions(cachedImagePath, dims); + } + } else { + LOG_ERR("EHP", "Failed to extract image"); } - gotDimensions = decoder && decoder->getDimensions(cachedImagePath, dims); } + if (gotDimensions) { LOG_DBG("EHP", "Image dimensions: %dx%d", dims.width, dims.height); @@ -722,7 +742,8 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->currentPageNextY += imageMarginTop; // Create ImageBlock and add to page - auto imageBlock = std::make_shared(cachedImagePath, displayWidth, displayHeight); + auto imageBlock = + std::make_shared(cachedImagePath, resolvedPath, displayWidth, displayHeight); if (!imageBlock) { LOG_ERR("EHP", "Failed to create ImageBlock"); return; @@ -753,8 +774,6 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* LOG_ERR("EHP", "Failed to get image dimensions"); Storage.remove(cachedImagePath.c_str()); } - } else { - LOG_ERR("EHP", "Failed to extract image"); } } // isFormatSupported } diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h index 9cc01e69..9c9c5afe 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h @@ -28,6 +28,7 @@ class ChapterHtmlSlimParser { GfxRenderer& renderer; std::function, uint16_t, uint16_t)> completePageFn; std::function popupFn; // Popup callback + bool imagePopupFired = false; // popupFn fired for the first image probe (single-shot) int depth = 0; int skipUntilDepth = INT_MAX; int boldUntilDepth = INT_MAX; diff --git a/lib/ZipFile/ZipFile.cpp b/lib/ZipFile/ZipFile.cpp index 867a155c..e18c2a4a 100644 --- a/lib/ZipFile/ZipFile.cpp +++ b/lib/ZipFile/ZipFile.cpp @@ -437,7 +437,7 @@ uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const boo return data; } -bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t chunkSize) { +bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t chunkSize, const bool allowEarlyStop) { const ScopedOpenClose zip{*this}; if (!zip) return false; @@ -469,8 +469,9 @@ bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t ch } if (out.write(buffer, dataRead) != dataRead) { - LOG_ERR("ZIP", "Failed to write all output bytes to stream"); free(buffer); + if (allowEarlyStop) return true; // sink has what it needs + LOG_ERR("ZIP", "Failed to write all output bytes to stream"); return false; } remaining -= dataRead; @@ -525,7 +526,11 @@ bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t ch if (produced > 0) { if (out.write(outputBuffer, produced) != produced) { - LOG_ERR("ZIP", "Failed to write all output bytes to stream"); + if (allowEarlyStop) { + success = true; // sink has what it needs + } else { + LOG_ERR("ZIP", "Failed to write all output bytes to stream"); + } break; } } diff --git a/lib/ZipFile/ZipFile.h b/lib/ZipFile/ZipFile.h index 157c3920..7052e524 100644 --- a/lib/ZipFile/ZipFile.h +++ b/lib/ZipFile/ZipFile.h @@ -69,7 +69,10 @@ class ZipFile { // Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple // These functions will open and close the zip as needed uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false); - bool readFileToStream(const char* filename, Print& out, size_t chunkSize); + // allowEarlyStop: a short write from `out` is treated as the sink asking to + // stop (returns true) instead of a write failure — used by header probes + // that only need the first bytes of an entry. + bool readFileToStream(const char* filename, Print& out, size_t chunkSize, bool allowEarlyStop = false); template bool enumerateFilePaths(F&& callback) { @@ -80,6 +83,14 @@ class ZipFile { return true; } + return enumerateFileEntries([&callback](std::string_view path, uint32_t, uint32_t) { callback(path); }); + } + + // Callback receives (path, crc32, compressedSize) for each central-directory + // entry. Always scans the central directory: the slim-stat cache does not + // hold CRCs. + template + bool enumerateFileEntries(F&& callback) { const bool wasOpen = isOpen(); if (!wasOpen && !open()) { return false; @@ -103,7 +114,11 @@ class ZipFile { break; } - file.seekCur(24); + file.seekCur(12); + uint32_t crc32, compressedSize; + file.read(&crc32, 4); + file.read(&compressedSize, 4); + file.seekCur(4); uint16_t nameLen, m, k; file.read(&nameLen, 2); file.read(&m, 2); @@ -113,7 +128,7 @@ class ZipFile { if (nameLen < sizeof(itemName)) { file.read(itemName, nameLen); itemName[nameLen] = '\0'; - callback(std::string_view{itemName, nameLen}); + callback(std::string_view{itemName, nameLen}, crc32, compressedSize); } else { file.seekCur(nameLen); } diff --git a/src/activities/reader/EpubReaderActivity.cpp b/src/activities/reader/EpubReaderActivity.cpp index 6057b13a..d49e6a2a 100644 --- a/src/activities/reader/EpubReaderActivity.cpp +++ b/src/activities/reader/EpubReaderActivity.cpp @@ -157,6 +157,11 @@ void EpubReaderActivity::onEnter() { } ImageBlock::clearSessionRenderFailures(); + // Lazy image extraction: section builds only header-probe images, so the first + // render of an image page pulls the file out of the EPUB through this hook. + ImageBlock::setExtractor(epub.get(), [](void* ctx, const char* src, const char* dest) { + return static_cast(ctx)->extractItemToFile(src, dest); + }); // Configure screen orientation based on settings // NOTE: This affects layout math and must be applied before any render calls. @@ -209,6 +214,10 @@ void EpubReaderActivity::onEnter() { void EpubReaderActivity::onExit() { Activity::onExit(); + // The extractor holds a raw pointer to this activity's epub; drop it before + // the activity (and the shared_ptr) goes away. + ImageBlock::setExtractor(nullptr, nullptr); + // Reset orientation back to portrait for the rest of the UI renderer.setOrientation(GfxRenderer::Orientation::Portrait); @@ -270,6 +279,18 @@ bool EpubReaderActivity::buildTickHeapGate() { return !buildHeapPaused; } +void EpubReaderActivity::showBuildPopup() { + // Mid-build indexing popup: only during onEnter's blocking build-to-target phase + // (buildPopupPending), at most once, and only when the framebuffer isn't on loan. + // If it fires while the loan is active (e.g. the parser's size-based call during + // startBuild), pending stays set and the deadline check retries after the loan. + if (!buildPopupPending || !renderer.hasFrameBuffer()) return; + GUI.drawPopup(renderer, tr(STR_INDEXING)); + // HALF-clear the popup when the page replaces it, else "INDEXING" ghosts. + pagesUntilFullRefresh = 1; + buildPopupPending = false; +} + void EpubReaderActivity::openDictionaryWordSelect() { if (SETTINGS.dictionaryName[0] == '\0') { showDictionaryMessage = true; @@ -1183,18 +1204,29 @@ void EpubReaderActivity::render(RenderLock&& lock) { // HALF-clear the popup when the page replaces it, else "INDEXING" ghosts under the page. pagesUntilFullRefresh = 1; } - // Lend the framebuffer's 48 KB to the blocking pre-render burst - // (startBuild inflates the whole spine HTML — the memory peak). The - // background buildSomeMore chunks in loop() do NOT get the loan: they - // deliberately interleave with page renders. Restored before render. - GfxRenderer::FrameBufferLoan loan(renderer); - if (!section->startBuild(SETTINGS.getReaderFontId(), SETTINGS.getReaderLineCompression(), - SETTINGS.extraParagraphSpacing, SETTINGS.paragraphAlignment, viewportWidth, - viewportHeight, SETTINGS.hyphenationEnabled, SETTINGS.embeddedStyle, - SETTINGS.imageRendering, SETTINGS.focusReadingEnabled)) { + // Mid-build popup surfacing for slow builds the predictive gates can't + // see (image extraction/probing inside a single page, or any chunk + // overrunning the deadline). The parser fires the callback before the + // first image probe; buildPopupPending gates it to this blocking phase + // so a background build in loop() can never draw over a displayed page. + buildPopupPending = !showPopup; + const unsigned long buildStartMs = millis(); + bool started; + { + // Lend the framebuffer's 48 KB to startBuild only (the spine HTML + // inflation peak). The chunk loop below runs without it so the popup + // can draw mid-build; background chunks never had the loan either. + GfxRenderer::FrameBufferLoan loan(renderer); + started = section->startBuild(SETTINGS.getReaderFontId(), SETTINGS.getReaderLineCompression(), + SETTINGS.extraParagraphSpacing, SETTINGS.paragraphAlignment, viewportWidth, + viewportHeight, SETTINGS.hyphenationEnabled, SETTINGS.embeddedStyle, + SETTINGS.imageRendering, SETTINGS.focusReadingEnabled, + [this] { showBuildPopup(); }); + } + if (!started) { LOG_ERR("ERS", "Failed to start section build"); section.reset(); - loan.end(); // restore before anything draws (showBuildError renders a popup) + buildPopupPending = false; showBuildError(); return; } @@ -1203,15 +1235,19 @@ void EpubReaderActivity::render(RenderLock&& lock) { // Anchor jump: build until the anchor's page is laid out (usually page 0), checking a // partial's on-disk anchor map too so an already-indexed anchor resolves immediately. // Otherwise: build until the target page exists. loop() builds the rest behind it. + if (buildPopupPending && millis() - buildStartMs >= BUILD_POPUP_DEADLINE_MS) { + // The predictive gates guessed fast but the build blew the silent budget. + showBuildPopup(); + } if (!section->buildSomeMore(BUILD_PAGES_PER_CHUNK)) { LOG_ERR("ERS", "Failed during incremental section build"); section.reset(); - loan.end(); // restore before anything draws (showBuildError renders a popup) + buildPopupPending = false; showBuildError(); return; } } - loan.end(); + buildPopupPending = false; } } } else { diff --git a/src/activities/reader/EpubReaderActivity.h b/src/activities/reader/EpubReaderActivity.h index a6dbf1cc..45f60a1c 100644 --- a/src/activities/reader/EpubReaderActivity.h +++ b/src/activities/reader/EpubReaderActivity.h @@ -146,6 +146,16 @@ class EpubReaderActivity final : public Activity { // whole HTML must be inflated before page 1 can lay out (the giant single-spine case), which is // a multi-second wait. Normal chapters are well under this and stay popup-free. static constexpr size_t BUILD_POPUP_BYTE_THRESHOLD = 96 * 1024; + // Deadline backstop for the predictive gates above: if the blocking build-to-target still + // hasn't produced the landing page this long after the build started, surface the popup + // mid-build. Builds that finish under the deadline stay popup-free. + static constexpr unsigned long BUILD_POPUP_DEADLINE_MS = 1000; + // True only during onEnter's blocking build-to-target phase, until the popup has been + // drawn. Gates showBuildPopup() so the parser's popup callback (which persists into + // background buildSomeMore chunks) can never draw over a displayed page. + bool buildPopupPending = false; + // Draw the indexing popup mid-build (parser image-probe callback and deadline backstop). + void showBuildPopup(); // Remap the cached relative reading position once the section's real page count is known // (used after a settings change re-paginates a chapter). Returns true if currentPage moved. // No-op while the section is still building or when the pagination is unchanged (plain resume).