diff --git a/docs/file-formats.md b/docs/file-formats.md index 36ded6e5..61c4c8bd 100644 --- a/docs/file-formats.md +++ b/docs/file-formats.md @@ -104,7 +104,7 @@ if (parsedSize != fileSize) { ## `section.bin` -### Version 20 +### Version 21 ImHex Pattern: @@ -114,7 +114,7 @@ import std.string; import std.core; // === Configuration === -#define EXPECTED_VERSION 20 +#define EXPECTED_VERSION 21 #define MAX_STRING_LENGTH 65535 // === String Structure === @@ -206,7 +206,7 @@ struct SectionBin { u8 imageRendering; u32 pageLutOffset [[comment("Offset to page offset LUT")]]; u32 anchorMapOffset [[comment("Offset to anchor map")]]; - u32 paragraphLutOffset [[comment("Offset to per-page paragraph index LUT")]]; + u32 paragraphLutOffset [[comment("Offset to per-page paragraph LUT (byte offset +

index)")]]; Page page[pageCount]; diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index bdf6c0a5..537348bb 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -27,6 +27,12 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION sizeof(uint32_t) + // page LUT offset sizeof(uint32_t) + // anchor map offset sizeof(uint32_t); // paragraph LUT offset + +// On-disk paragraph LUT entry: u32 xhtmlByteOffset + u16 paragraphIndex. +constexpr uint32_t PARAGRAPH_LUT_ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t); +inline uint32_t paragraphLutEntryOffset(uint32_t lutStart, uint16_t page) { + return lutStart + page * PARAGRAPH_LUT_ENTRY_SIZE; +} } // namespace uint32_t Section::onPageComplete(std::unique_ptr page) { @@ -551,36 +557,43 @@ std::optional Section::getPageForAnchor(const std::string& anchor) con return std::nullopt; } +bool Section::readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32_t& outLutStart) const { + if (!Storage.openFileForRead("SCT", filePath, outFile)) { + return false; + } + + const uint32_t fileSize = outFile.size(); + + outFile.seek(HEADER_SIZE - sizeof(uint32_t)); + uint32_t paragraphLutOffset; + serialization::readPod(outFile, paragraphLutOffset); + if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) { + outFile.close(); + return false; + } + + outFile.seek(paragraphLutOffset); + serialization::readPod(outFile, outCount); + if (outCount == 0) { + outFile.close(); + return false; + } + + outLutStart = paragraphLutOffset + sizeof(uint16_t); + const uint32_t lutEnd = outLutStart + outCount * PARAGRAPH_LUT_ENTRY_SIZE; + if (lutEnd > fileSize) { + outFile.close(); + return false; + } + + return true; +} + std::optional Section::getPageForParagraphIndex(const uint16_t pIndex) const { FsFile f; - if (!Storage.openFileForRead("SCT", filePath, f)) { - return std::nullopt; - } - - const uint32_t fileSize = f.size(); - - // Read paragraph LUT offset from end of header - f.seek(HEADER_SIZE - sizeof(uint32_t)); - uint32_t paragraphLutOffset; - serialization::readPod(f, paragraphLutOffset); - if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) { - f.close(); - return std::nullopt; - } - - f.seek(paragraphLutOffset); - uint16_t count; - serialization::readPod(f, count); - if (count == 0) { - f.close(); - return std::nullopt; - } - - // Each entry: uint32_t xhtmlByteOffset + uint16_t paragraphIndex - constexpr uint32_t ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t); - const uint32_t lutEnd = paragraphLutOffset + sizeof(uint16_t) + count * ENTRY_SIZE; - if (lutEnd > fileSize) { - f.close(); + uint16_t count = 0; + uint32_t lutStart = 0; + if (!readParagraphLutHeader(f, count, lutStart)) { return std::nullopt; } @@ -588,7 +601,7 @@ std::optional Section::getPageForParagraphIndex(const uint16_t pIndex) //

whose start tag had been seen while page i was being laid out. Paragraph // P therefore first appears on the smallest i where storedPIdx[i] >= P. for (uint16_t i = 0; i < count; i++) { - f.seek(paragraphLutOffset + sizeof(uint16_t) + i * ENTRY_SIZE + sizeof(uint32_t)); + f.seek(paragraphLutEntryOffset(lutStart, i) + sizeof(uint32_t)); uint16_t pagePIdx; serialization::readPod(f, pagePIdx); if (pagePIdx >= pIndex) { @@ -603,38 +616,18 @@ std::optional Section::getPageForParagraphIndex(const uint16_t pIndex) std::optional Section::getParagraphIndexForPage(const uint16_t page) const { FsFile f; - if (!Storage.openFileForRead("SCT", filePath, f)) { + uint16_t count = 0; + uint32_t lutStart = 0; + if (!readParagraphLutHeader(f, count, lutStart)) { return std::nullopt; } - - const uint32_t fileSize = f.size(); - - f.seek(HEADER_SIZE - sizeof(uint32_t)); - uint32_t paragraphLutOffset; - serialization::readPod(f, paragraphLutOffset); - if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) { - f.close(); - return std::nullopt; - } - - f.seek(paragraphLutOffset); - uint16_t count; - serialization::readPod(f, count); - if (count == 0 || page >= count) { - f.close(); - return std::nullopt; - } - - // Each entry: uint32_t xhtmlByteOffset + uint16_t paragraphIndex - constexpr uint32_t ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t); - const uint32_t entryEnd = paragraphLutOffset + sizeof(uint16_t) + (page + 1) * ENTRY_SIZE; - if (entryEnd > fileSize) { + if (page >= count) { f.close(); return std::nullopt; } // Seek directly to the paragraphIndex field of the requested entry (skip xhtmlByteOffset) - f.seek(paragraphLutOffset + sizeof(uint16_t) + page * ENTRY_SIZE + sizeof(uint32_t)); + f.seek(paragraphLutEntryOffset(lutStart, page) + sizeof(uint32_t)); uint16_t pIdx; serialization::readPod(f, pIdx); @@ -644,36 +637,17 @@ std::optional Section::getParagraphIndexForPage(const uint16_t page) c std::optional Section::getXhtmlByteOffsetForPage(const uint16_t page) const { FsFile f; - if (!Storage.openFileForRead("SCT", filePath, f)) { + uint16_t count = 0; + uint32_t lutStart = 0; + if (!readParagraphLutHeader(f, count, lutStart)) { return std::nullopt; } - - const uint32_t fileSize = f.size(); - - f.seek(HEADER_SIZE - sizeof(uint32_t)); - uint32_t paragraphLutOffset; - serialization::readPod(f, paragraphLutOffset); - if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) { + if (page >= count) { f.close(); return std::nullopt; } - f.seek(paragraphLutOffset); - uint16_t count; - serialization::readPod(f, count); - if (count == 0 || page >= count) { - f.close(); - return std::nullopt; - } - - constexpr uint32_t ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t); - const uint32_t entryEnd = paragraphLutOffset + sizeof(uint16_t) + (page + 1) * ENTRY_SIZE; - if (entryEnd > fileSize) { - f.close(); - return std::nullopt; - } - - f.seek(paragraphLutOffset + sizeof(uint16_t) + page * ENTRY_SIZE); + f.seek(paragraphLutEntryOffset(lutStart, page)); uint32_t byteOffset; serialization::readPod(f, byteOffset); diff --git a/lib/Epub/Epub/Section.h b/lib/Epub/Epub/Section.h index d50d8c35..d2c85b4c 100644 --- a/lib/Epub/Epub/Section.h +++ b/lib/Epub/Epub/Section.h @@ -32,6 +32,12 @@ class Section { void buildTocBoundaries(const std::vector>& anchors); void buildTocBoundariesFromFile(FsFile& f); + // Open the section file and seek to the first paragraph LUT entry, validating the header + // and LUT bounds against fileSize. On success, returns true with `outLutStart` set to the + // byte offset of the first entry (just past the count) and `outCount` to the entry count. + // Caller is responsible for closing `outFile`. Returns false on any I/O or validation error. + bool readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32_t& outLutStart) const; + public: uint16_t pageCount = 0; int currentPage = 0; diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index 2ccfd89a..8f520fae 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -593,9 +593,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* (self->currentPageNextY + totalImageHeightWithSpacing > self->viewportHeight)) { LOG_DBG("EHP", "Image page break: currentY=%d needed=%d viewportH=%d", self->currentPageNextY, totalImageHeightWithSpacing, self->viewportHeight); - const uint32_t byteOff = - self->activeParser ? static_cast(XML_GetCurrentByteIndex(self->activeParser)) : 0; - self->paragraphLutPerPage.push_back({byteOff, self->xpathParagraphIndex}); + self->paragraphLutPerPage.push_back({self->lastBodyChildByteOffset, self->xpathParagraphIndex}); self->completePageFn(std::move(self->currentPage)); self->completedPageCount++; self->currentPage.reset(new Page()); @@ -689,8 +687,16 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* // check so that hidden

elements are still counted, matching ChapterXPathIndexer's // counting (pure XML, no CSS). This ensures paragraph indices in the section cache LUT // align with KOReader's crengine XPath indices. - if (self->xpathBodyDepth >= 0 && self->depth == self->xpathBodyDepth + 1 && strcmp(name, "p") == 0) { - self->xpathParagraphIndex++; + // At the same time, record the byte offset of every direct-body-child element start: + // the forward mapper's partial-parse heuristic requires the seek hint to land on a + // body-child boundary, otherwise partialBaseDepth can misidentify wrapped paragraphs. + if (self->xpathBodyDepth >= 0 && self->depth == self->xpathBodyDepth + 1) { + if (self->activeParser) { + self->lastBodyChildByteOffset = static_cast(XML_GetCurrentByteIndex(self->activeParser)); + } + if (strcmp(name, "p") == 0) { + self->xpathParagraphIndex++; + } } if (matches(name, SKIP_TAGS, NUM_SKIP_TAGS)) { @@ -1438,8 +1444,7 @@ ParsedText::LineProcessResult ChapterHtmlSlimParser::addLineToPage(std::shared_p } if (currentPageNextY + lineHeight > viewportHeight) { - const uint32_t byteOff = activeParser ? static_cast(XML_GetCurrentByteIndex(activeParser)) : 0; - paragraphLutPerPage.push_back({byteOff, xpathParagraphIndex}); + paragraphLutPerPage.push_back({lastBodyChildByteOffset, xpathParagraphIndex}); completePageFn(std::move(currentPage)); completedPageCount++; currentPage.reset(new Page()); diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h index 6aa01dc9..8d189f43 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h @@ -92,9 +92,15 @@ class ChapterHtmlSlimParser { // without reparsing, and current page can generate an XPath without reparsing. uint16_t xpathParagraphIndex = 0; // current

sibling index (1-based) int xpathBodyDepth = -1; // depth of the element (-1 = not yet seen) + // Byte offset of the most recent direct-body-child element start (any tag at xpathBodyDepth+1). + // Recorded at the same depth condition that increments xpathParagraphIndex, so the stored + // offset is guaranteed to land on a body-child element boundary. This keeps the XPath forward + // mapper's partial-parse heuristic reliable for wrapped chapters: without this, the offset + // could point mid-way into a nested

/
, which confuses partialBaseDepth. + uint32_t lastBodyChildByteOffset = 0; struct ParagraphLutEntry { - uint32_t xhtmlByteOffset; // Expat byte offset at page break — used to seek near target paragraph + uint32_t xhtmlByteOffset; // byte offset of most recent body-child element start at page break uint16_t paragraphIndex; // 1-based

index at page completion }; std::vector paragraphLutPerPage; // deep LUT: one entry per page diff --git a/lib/KOReaderSync/ChapterXPathForwardMapper.cpp b/lib/KOReaderSync/ChapterXPathForwardMapper.cpp index c77bfd4e..d6fc51ec 100644 --- a/lib/KOReaderSync/ChapterXPathForwardMapper.cpp +++ b/lib/KOReaderSync/ChapterXPathForwardMapper.cpp @@ -223,7 +223,10 @@ std::string findXPathForParagraphInternal(const std::shared_ptr& epub, con // Partial parse missed — reset and retry from beginning with full-document context. XML_ParserFree(parser); parser = XML_ParserCreate(nullptr); - if (parser) { + if (!parser) { + LOG_ERR("KOX", "XML_ParserCreate failed on retry: spine=%d p[%u] tmp=%s", spineIndex, paragraphIndex, + tmpPath.c_str()); + } else { ParagraphState fullState(spineIndex, paragraphIndex, 0, false); fullState.parser = parser; XML_SetUserData(parser, &fullState); diff --git a/lib/KOReaderSync/ChapterXPathIndexer.h b/lib/KOReaderSync/ChapterXPathIndexer.h index 9854de00..3ee718f3 100644 --- a/lib/KOReaderSync/ChapterXPathIndexer.h +++ b/lib/KOReaderSync/ChapterXPathIndexer.h @@ -76,6 +76,12 @@ class ChapterXPathIndexer { * @param paragraphIndex 1-based paragraph index (from section LUT or XPath p[N]) * @param seekHint Optional XHTML byte offset to start scanning from (0 = from beginning). * Pass Section::getXhtmlByteOffsetForPage() to avoid scanning the whole file. + * @param startParagraphCount Optional seed count (default 0) of direct-body-child

elements + * that precede the seekHint position. Should be provided when seekHint > 0 to + * avoid counting from scratch mid-document; callers should pass the paragraph + * index of the LUT entry at the seek page minus 1. If the partial parse with + * this seed doesn't find the target, the function falls back to runParse from + * byte 0 and re-counts with startParagraphCount = 0. * @return Full-ancestry XPath like "/body/DocFragment[N]/body/div[1]/p[3]", or empty on failure */ static std::string findXPathForParagraph(const std::shared_ptr& epub, int spineIndex, uint16_t paragraphIndex, diff --git a/lib/KOReaderSync/ChapterXPathIndexerInternal.cpp b/lib/KOReaderSync/ChapterXPathIndexerInternal.cpp index f678baf0..fc5358f7 100644 --- a/lib/KOReaderSync/ChapterXPathIndexerInternal.cpp +++ b/lib/KOReaderSync/ChapterXPathIndexerInternal.cpp @@ -245,33 +245,43 @@ std::string decompressToTempFile(const std::shared_ptr& epub, const int sp return tmpPath; } +namespace { +// Pump the open `file` through `parser` in fixed-size chunks. Returns true on clean EOF or +// XML_ERROR_ABORTED (caller used XML_StopParser to signal an early success). Returns false on +// XML_GetBuffer failure or any other parse error. The file is left open — caller closes it. +bool pumpExpatFromFile(XML_Parser parser, FsFile& file) { + constexpr size_t kBufSize = 1024; + int done; + do { + void* const buf = XML_GetBuffer(parser, kBufSize); + if (!buf) { + return false; + } + const size_t len = file.read(buf, kBufSize); + done = file.available() == 0; + if (XML_ParseBuffer(parser, static_cast(len), done) == XML_STATUS_ERROR) { + return XML_GetErrorCode(parser) == XML_ERROR_ABORTED; + } + } while (!done); + return true; +} +} // namespace + bool runParse(XML_Parser parser, const std::string& path) { FsFile file; if (!Storage.openFileForRead("KOX", path, file)) { return false; } - - constexpr size_t kBufSize = 1024; - bool ok = true; - int done; - do { - void* const buf = XML_GetBuffer(parser, kBufSize); - if (!buf) { - ok = false; - break; - } - const size_t len = file.read(buf, kBufSize); - done = file.available() == 0; - if (XML_ParseBuffer(parser, static_cast(len), done) == XML_STATUS_ERROR) { - ok = (XML_GetErrorCode(parser) == XML_ERROR_ABORTED); - break; - } - } while (!done); - + const bool ok = pumpExpatFromFile(parser, file); file.close(); return ok; } +// Starts Expat mid-document. Since the parser has no ancestor context (html/body stack is +// missing), unmatched closing tags may appear, and callbacks emitted before the first start +// tag can look structurally odd — an empty result is normal here. Callers +// (ChapterXPathForwardMapper.cpp) recognise the empty result and fall back to runParse from +// byte 0 with full document context. bool runParseFromOffset(XML_Parser parser, const std::string& path, const uint32_t seekBytes) { if (seekBytes == 0) { return runParse(parser, path); @@ -287,23 +297,7 @@ bool runParseFromOffset(XML_Parser parser, const std::string& path, const uint32 return runParse(parser, path); // fall back to full scan if seek fails } - constexpr size_t kBufSize = 1024; - bool ok = true; - int done; - do { - void* const buf = XML_GetBuffer(parser, kBufSize); - if (!buf) { - ok = false; - break; - } - const size_t len = file.read(buf, kBufSize); - done = file.available() == 0; - if (XML_ParseBuffer(parser, static_cast(len), done) == XML_STATUS_ERROR) { - ok = (XML_GetErrorCode(parser) == XML_ERROR_ABORTED); - break; - } - } while (!done); - + const bool ok = pumpExpatFromFile(parser, file); file.close(); return ok; } diff --git a/lib/KOReaderSync/ProgressMapper.cpp b/lib/KOReaderSync/ProgressMapper.cpp index 0d2c0fcf..eaf7e7ea 100644 --- a/lib/KOReaderSync/ProgressMapper.cpp +++ b/lib/KOReaderSync/ProgressMapper.cpp @@ -63,8 +63,14 @@ KOReaderPosition ProgressMapper::toKOReader(const std::shared_ptr& epub, c // where paragraphs are nested inside divs/sections. Fall back to the progress-based // scan (which works for any content) when no paragraph index is available. if (pos.hasParagraphIndex && pos.paragraphIndex > 0) { - result.xpath = - ChapterXPathIndexer::findXPathForParagraph(epub, pos.spineIndex, pos.paragraphIndex, pos.xhtmlSeekHint); + // When a seek hint is set, the LUT entry's paragraphIndex equals pos.paragraphIndex + // (both describe the same page). The byte offset now points at the body-child element + // that was current at the page break, so re-parsing from there will re-encounter that + // paragraph — seed startParagraphCount with paragraphIndex-1 to avoid double counting. + const uint16_t startCount = + pos.xhtmlSeekHint > 0 && pos.paragraphIndex > 0 ? static_cast(pos.paragraphIndex - 1) : 0; + result.xpath = ChapterXPathIndexer::findXPathForParagraph(epub, pos.spineIndex, pos.paragraphIndex, + pos.xhtmlSeekHint, startCount); } if (result.xpath.empty()) { result.xpath = ChapterXPathIndexer::findXPathForProgress(epub, pos.spineIndex, intraSpineProgress); diff --git a/src/JsonSettingsIO.cpp b/src/JsonSettingsIO.cpp index b09692d0..a91495a0 100644 --- a/src/JsonSettingsIO.cpp +++ b/src/JsonSettingsIO.cpp @@ -82,6 +82,7 @@ bool JsonSettingsIO::saveState(const CrossPointState& s, const char* path) { sync["totalPagesInSpine"] = s.koReaderSyncSession.totalPagesInSpine; sync["paragraphIndex"] = s.koReaderSyncSession.paragraphIndex; sync["hasParagraphIndex"] = s.koReaderSyncSession.hasParagraphIndex; + sync["xhtmlSeekHint"] = s.koReaderSyncSession.xhtmlSeekHint; sync["intent"] = static_cast(s.koReaderSyncSession.intent); sync["outcome"] = static_cast(s.koReaderSyncSession.outcome); sync["resultSpineIndex"] = s.koReaderSyncSession.resultSpineIndex; @@ -134,6 +135,7 @@ bool JsonSettingsIO::loadState(CrossPointState& s, const char* json) { s.koReaderSyncSession.totalPagesInSpine = sync["totalPagesInSpine"] | 0; s.koReaderSyncSession.paragraphIndex = sync["paragraphIndex"] | (uint16_t)0; s.koReaderSyncSession.hasParagraphIndex = sync["hasParagraphIndex"] | false; + s.koReaderSyncSession.xhtmlSeekHint = sync["xhtmlSeekHint"] | (uint32_t)0; s.koReaderSyncSession.intent = static_cast(sync["intent"] | static_cast(KOReaderSyncIntentState::COMPARE)); s.koReaderSyncSession.outcome =