diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index 98e2c162..157f35dd 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -191,7 +191,6 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c const uint8_t imageRendering, const std::function& progressFn) { const uint32_t phaseTotalStart = millis(); const auto localPath = epub->getSpineItem(spineIndex).href; - const auto tmpHtmlPath = epub->getCachePath() + "/.tmp_" + std::to_string(spineIndex) + ".html"; // Create cache directory if it doesn't exist { @@ -199,45 +198,14 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c Storage.mkdir(sectionsDir.c_str()); } - // Retry logic for SD card timing issues - const uint32_t phaseStreamStart = millis(); - bool success = false; - uint32_t fileSize = 0; - for (int attempt = 0; attempt < 3 && !success; attempt++) { - if (attempt > 0) { - LOG_DBG("SCT", "Retrying stream (attempt %d)...", attempt + 1); - delay(50); // Brief delay before retry - } - - // Remove any incomplete file from previous attempt before retrying - if (Storage.exists(tmpHtmlPath.c_str())) { - Storage.remove(tmpHtmlPath.c_str()); - } - - FsFile tmpHtml; - if (!Storage.openFileForWrite("SCT", tmpHtmlPath, tmpHtml)) { - continue; - } - success = epub->readItemContentsToStream(localPath, tmpHtml, 1024); - fileSize = tmpHtml.size(); - tmpHtml.close(); - - // If streaming failed, remove the incomplete file immediately - if (!success && Storage.exists(tmpHtmlPath.c_str())) { - Storage.remove(tmpHtmlPath.c_str()); - LOG_DBG("SCT", "Removed incomplete temp file after failed attempt"); - } - } - - if (!success) { - LOG_ERR("SCT", "Failed to stream item contents to temp file after retries"); + // Get inflated size up-front so the parser can choose progress granularity. + const uint32_t phaseSetupStart = millis(); + size_t inflatedSize = 0; + if (!epub->getItemSize(localPath, &inflatedSize)) { + LOG_ERR("SCT", "Failed to get inflated size for %s", localPath.c_str()); return false; } - const uint32_t streamMs = millis() - phaseStreamStart; - LOG_DBG("SCT", "Streamed temp HTML to %s (%d bytes)", tmpHtmlPath.c_str(), fileSize); - - const uint32_t phaseSetupStart = millis(); if (!Storage.openFileForWrite("SCT", filePath, file)) { return false; } @@ -274,20 +242,14 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c } ChapterHtmlSlimParser visitor( - epub, tmpHtmlPath, renderer, fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth, - viewportHeight, hyphenationEnabled, + epub, renderer, fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth, viewportHeight, + hyphenationEnabled, [this, &lut](std::unique_ptr page) { lut.emplace_back(this->onPageComplete(std::move(page))); }, embeddedStyle, contentBase, imageBasePath, imageRendering, std::move(tocAnchors), progressFn, cssParser); Hyphenator::setPreferredLanguage(epub->getLanguage()); - const uint32_t setupMs = millis() - phaseSetupStart; - const uint32_t phaseParseStart = millis(); - success = visitor.parseAndBuildPages(); - const uint32_t parseMs = millis() - phaseParseStart; - const uint32_t phaseFinalizeStart = millis(); - Storage.remove(tmpHtmlPath.c_str()); - if (!success) { - LOG_ERR("SCT", "Failed to parse XML and build pages"); + if (!visitor.setup(inflatedSize)) { + LOG_ERR("SCT", "Failed to set up chapter parser"); file.close(); Storage.remove(filePath.c_str()); if (cssParser) { @@ -295,6 +257,29 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c } return false; } + const uint32_t setupMs = millis() - phaseSetupStart; + + // Stream EPUB item content directly into the parser — no temp file, no second SD pass. + const uint32_t phaseParseStart = millis(); + const bool streamOk = epub->readItemContentsToStream(localPath, visitor, 1024); + const bool finalizeOk = visitor.finalize(); + bool success = streamOk && finalizeOk && visitor.streamSucceeded(); + const uint32_t parseMs = millis() - phaseParseStart; + // streamMs is no longer a separate phase (SD-write of temp file is gone); keep the + // log breakdown stable by reporting it as 0. + constexpr uint32_t streamMs = 0; + + const uint32_t phaseFinalizeStart = millis(); + if (!success) { + LOG_ERR("SCT", "Failed to parse XML and build pages (stream=%d finalize=%d)", streamOk ? 1 : 0, finalizeOk ? 1 : 0); + file.close(); + Storage.remove(filePath.c_str()); + if (cssParser) { + cssParser->clear(); + } + return false; + } + const uint32_t fileSize = static_cast(inflatedSize); const uint32_t lutOffset = file.position(); bool hasFailedLutRecords = false; diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index dfcfcebe..5b2ba0ab 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -22,11 +22,11 @@ constexpr int NUM_HEADER_TAGS = sizeof(HEADER_TAGS) / sizeof(HEADER_TAGS[0]); // Size thresholds (bytes of XHTML) controlling indexing popup behavior. // Each progress callback costs ~640ms of e-ink refresh, so we trade granularity off // against indexing time based on expected duration. -// < 10KB: no popup at all - indexing finishes faster than the popup would draw +// < 15KB: no popup at all - indexing finishes faster than the popup would draw // < 30KB: popup only (one refresh up-front, no mid-parse updates) // < 80KB: popup + one heartbeat at 50% // >= 80KB: popup + ticks at 25/50/75% -constexpr size_t MIN_SIZE_FOR_POPUP = 10 * 1024; +constexpr size_t MIN_SIZE_FOR_POPUP = 15 * 1024; constexpr size_t SIZE_FOR_PROGRESS_HEARTBEAT = 30 * 1024; constexpr size_t SIZE_FOR_PROGRESS_FINE = 80 * 1024; constexpr size_t PARSE_BUFFER_SIZE = 1024; @@ -1321,7 +1321,17 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n } } -bool ChapterHtmlSlimParser::parseAndBuildPages() { +ChapterHtmlSlimParser::~ChapterHtmlSlimParser() { + if (activeParser) { + XML_StopParser(activeParser, XML_FALSE); + XML_SetElementHandler(activeParser, nullptr, nullptr); + XML_SetCharacterDataHandler(activeParser, nullptr); + XML_ParserFree(activeParser); + activeParser = nullptr; + } +} + +bool ChapterHtmlSlimParser::setup(const size_t totalInflatedSize) { auto paragraphAlignmentBlockStyle = BlockStyle(); paragraphAlignmentBlockStyle.textAlignDefined = true; // Resolve None sentinel to Justify for initial block (no CSS context yet) @@ -1331,113 +1341,117 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() { paragraphAlignmentBlockStyle.alignment = align; startNewTextBlock(paragraphAlignmentBlockStyle); - const XML_Parser parser = XML_ParserCreate(nullptr); - int done; - + XML_Parser parser = XML_ParserCreate(nullptr); if (!parser) { LOG_ERR("EHP", "Couldn't allocate memory for parser"); return false; } - // Handle HTML entities (like  ) that aren't in XML spec or DTD - // Using DefaultHandlerExpand preserves normal entity expansion from DOCTYPE + // Handle HTML entities (like  ) that aren't in XML spec or DTD. + // Using DefaultHandlerExpand preserves normal entity expansion from DOCTYPE. XML_SetDefaultHandlerExpand(parser, defaultHandlerExpand); - - FsFile file; - if (!Storage.openFileForRead("EHP", filepath, file)) { - XML_ParserFree(parser); - return false; - } - - const size_t totalFileSize = file.size(); - size_t bytesRead = 0; - int lastReportedProgress = -1; - - // Choose progress granularity by chapter size. Each callback drives a full-screen - // e-ink refresh (~640ms), so smaller chapters skip mid-parse ticks entirely. - // progressStepPercent == 0 means "popup only, no mid-parse updates". - int progressStepPercent = 0; - if (totalFileSize >= SIZE_FOR_PROGRESS_FINE) { - progressStepPercent = 25; - } else if (totalFileSize >= SIZE_FOR_PROGRESS_HEARTBEAT) { - progressStepPercent = 50; - } - - // Show initial progress popup for files above threshold. - if (progressFn && totalFileSize >= MIN_SIZE_FOR_POPUP) { - progressFn(0); - } - XML_SetUserData(parser, this); XML_SetElementHandler(parser, startElement, endElement); XML_SetCharacterDataHandler(parser, characterData); activeParser = parser; - // Compute the time taken to parse and build pages - const uint32_t chapterStartTime = millis(); - do { - void* const buf = XML_GetBuffer(parser, PARSE_BUFFER_SIZE); + totalStreamSize = totalInflatedSize; + bytesStreamed = 0; + lastReportedProgress = -1; + streamFailed = false; + streamStartTimeMs = millis(); + + // Choose progress granularity by chapter size. Each callback drives a full-screen + // e-ink refresh (~640ms), so smaller chapters skip mid-parse ticks entirely. + // progressStepPercent == 0 means "popup only, no mid-parse updates". + progressStepPercent = 0; + if (totalStreamSize >= SIZE_FOR_PROGRESS_FINE) { + progressStepPercent = 25; + } else if (totalStreamSize >= SIZE_FOR_PROGRESS_HEARTBEAT) { + progressStepPercent = 50; + } + + // Show initial progress popup for files above threshold. + if (progressFn && totalStreamSize >= MIN_SIZE_FOR_POPUP) { + progressFn(0); + } + return true; +} + +size_t ChapterHtmlSlimParser::write(const uint8_t data) { return write(&data, 1); } + +size_t ChapterHtmlSlimParser::write(const uint8_t* buffer, const size_t size) { + if (!activeParser || streamFailed || size == 0) { + return streamFailed ? 0 : size; + } + + size_t remaining = size; + const uint8_t* cursor = buffer; + while (remaining > 0) { + const size_t chunk = remaining < PARSE_BUFFER_SIZE ? remaining : PARSE_BUFFER_SIZE; + void* const buf = XML_GetBuffer(activeParser, static_cast(chunk)); if (!buf) { - LOG_ERR("EHP", "Couldn't allocate memory for buffer"); - XML_StopParser(parser, XML_FALSE); // Stop any pending processing - XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks - XML_SetCharacterDataHandler(parser, nullptr); - activeParser = nullptr; - XML_ParserFree(parser); - file.close(); - return false; + LOG_ERR("EHP", "Couldn't allocate buffer"); + streamFailed = true; + return 0; + } + memcpy(buf, cursor, chunk); + + bytesStreamed += chunk; + // The streaming source doesn't know "this was the last chunk" — pass isFinal=false + // here and let finalize() emit the terminating empty parse with isFinal=true. + if (XML_ParseBuffer(activeParser, static_cast(chunk), 0) == XML_STATUS_ERROR) { + LOG_ERR("EHP", "Parse error at line %lu:\n%s", XML_GetCurrentLineNumber(activeParser), + XML_ErrorString(XML_GetErrorCode(activeParser))); + streamFailed = true; + return 0; } - const size_t len = file.read(buf, PARSE_BUFFER_SIZE); - bytesRead += len; + cursor += chunk; + remaining -= chunk; + } - // Report progress at the granularity chosen up-front (see progressStepPercent). - // Skip the 100% callback — the page render that follows immediately replaces the popup, - // so the final tick is wasted work. - if (progressFn && progressStepPercent > 0) { - const int progress = static_cast(bytesRead * 100 / totalFileSize); - if (progress < 100 && progress / progressStepPercent > lastReportedProgress / progressStepPercent) { - lastReportedProgress = progress; - progressFn(progress); - } + // Report progress at the granularity chosen up-front (see progressStepPercent). + // Skip the 100% callback — the page render that follows immediately replaces the popup, + // so the final tick is wasted work. + if (progressFn && progressStepPercent > 0 && totalStreamSize > 0) { + const int progress = static_cast(bytesStreamed * 100 / totalStreamSize); + if (progress < 100 && progress / progressStepPercent > lastReportedProgress / progressStepPercent) { + lastReportedProgress = progress; + progressFn(progress); } + } - if (len == 0 && file.available() > 0) { - LOG_ERR("EHP", "File read error"); - XML_StopParser(parser, XML_FALSE); // Stop any pending processing - XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks - XML_SetCharacterDataHandler(parser, nullptr); - activeParser = nullptr; - XML_ParserFree(parser); - file.close(); - return false; + return size; +} + +bool ChapterHtmlSlimParser::finalize() { + if (!activeParser) { + return false; + } + + bool success = !streamFailed; + if (success) { + // Emit terminating empty parse so Expat finalizes any pending tokens. + if (XML_ParseBuffer(activeParser, 0, 1) == XML_STATUS_ERROR) { + LOG_ERR("EHP", "Parse error at line %lu (finalize):\n%s", XML_GetCurrentLineNumber(activeParser), + XML_ErrorString(XML_GetErrorCode(activeParser))); + success = false; + streamFailed = true; } + } - done = file.available() == 0; + XML_StopParser(activeParser, XML_FALSE); + XML_SetElementHandler(activeParser, nullptr, nullptr); + XML_SetCharacterDataHandler(activeParser, nullptr); + XML_ParserFree(activeParser); + activeParser = nullptr; - if (XML_ParseBuffer(parser, static_cast(len), done) == XML_STATUS_ERROR) { - LOG_ERR("EHP", "Parse error at line %lu:\n%s", XML_GetCurrentLineNumber(parser), - XML_ErrorString(XML_GetErrorCode(parser))); - XML_StopParser(parser, XML_FALSE); // Stop any pending processing - XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks - XML_SetCharacterDataHandler(parser, nullptr); - activeParser = nullptr; - XML_ParserFree(parser); - file.close(); - return false; - } - } while (!done); - const uint32_t totalTimeMs = millis() - chapterStartTime; + const uint32_t totalTimeMs = millis() - streamStartTimeMs; LOG_DBG("EHP", "Time to parse and build pages: %lu ms", totalTimeMs); - XML_StopParser(parser, XML_FALSE); // Stop any pending processing - XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks - XML_SetCharacterDataHandler(parser, nullptr); - activeParser = nullptr; - XML_ParserFree(parser); - file.close(); - - // Process last page if there is still text + // Process last page if there is still text. Done unconditionally so that a partial + // success scenario still flushes whatever pages were produced. if (currentTextBlock) { makePages(); if (!pendingAnchorId.empty()) { @@ -1449,7 +1463,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() { currentTextBlock.reset(); } - return true; + return success; } ParsedText::LineProcessResult ChapterHtmlSlimParser::addLineToPage(std::shared_ptr line, diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h index 2ce5a1f5..1898a0b8 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -22,9 +23,8 @@ class Epub; #define MAX_WORD_SIZE 200 -class ChapterHtmlSlimParser { +class ChapterHtmlSlimParser final : public Print { std::shared_ptr epub; - const std::string& filepath; GfxRenderer& renderer; std::function)> completePageFn; std::function progressFn; // Progress callback (0-100) @@ -105,11 +105,19 @@ class ChapterHtmlSlimParser { }; std::vector paragraphLutPerPage; // deep LUT: one entry per page - // Active parser handle during parseAndBuildPages(), nullptr otherwise. + // Active parser handle during streaming, nullptr otherwise. // Stored as a member so page-break sites (addLineToPage, image breaks) can call // XML_GetCurrentByteIndex without needing the parser threaded through every call. XML_Parser activeParser = nullptr; + // Streaming state for the Print-derived parsing API. + size_t totalStreamSize = 0; + size_t bytesStreamed = 0; + int lastReportedProgress = -1; + int progressStepPercent = 0; + bool streamFailed = false; + uint32_t streamStartTimeMs = 0; + // Footnote link tracking bool insideFootnoteLink = false; int footnoteLinkDepth = -1; @@ -138,8 +146,8 @@ class ChapterHtmlSlimParser { static void XMLCALL endElement(void* userData, const XML_Char* name); public: - explicit ChapterHtmlSlimParser(std::shared_ptr epub, const std::string& filepath, GfxRenderer& renderer, - const int fontId, const float lineCompression, const bool extraParagraphSpacing, + explicit ChapterHtmlSlimParser(std::shared_ptr epub, GfxRenderer& renderer, const int fontId, + const float lineCompression, const bool extraParagraphSpacing, const uint8_t paragraphAlignment, const uint16_t viewportWidth, const uint16_t viewportHeight, const bool hyphenationEnabled, const std::function)>& completePageFn, @@ -150,7 +158,6 @@ class ChapterHtmlSlimParser { const CssParser* cssParser = nullptr) : epub(epub), - filepath(filepath), renderer(renderer), fontId(fontId), lineCompression(lineCompression), @@ -168,8 +175,22 @@ class ChapterHtmlSlimParser { imageBasePath(imageBasePath), tocAnchors(std::move(tocAnchors)) {} - ~ChapterHtmlSlimParser() = default; - bool parseAndBuildPages(); + ~ChapterHtmlSlimParser() override; + + // Streaming parse lifecycle. Caller flow: + // parser.setup(totalInflatedSize); + // epub->readItemContentsToStream(href, parser, ...); + // parser.finalize(); + // Returns false from setup() on parser allocation failure; check streamSucceeded() + // after finalize() to detect a parse error mid-stream. + bool setup(size_t totalInflatedSize); + bool finalize(); + [[nodiscard]] bool streamSucceeded() const { return !streamFailed; } + + // Print interface — fed by Epub::readItemContentsToStream. + size_t write(uint8_t) override; + size_t write(const uint8_t* buffer, size_t size) override; + ParsedText::LineProcessResult addLineToPage(std::shared_ptr line, bool lineEndsWithHyphenatedWord, bool suppressHyphenationRetry); const std::vector>& getAnchors() const { return anchorData; }