Merge pull request #109 from jpirnay/kosync_extension

refactor: Picking up some itsthisjustin ideas for KOreader synchronisation

lib/Epub/Epub.cpp

@@ -45,7 +45,7 @@ bool Epub::findContentOpfFile(std::string* contentOpfFile) const {
   return true;
 }
 
-bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
+bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, OpfCacheMode cacheMode) {
   std::string contentOpfFilePath;
   if (!findContentOpfFile(&contentOpfFilePath)) {
     LOG_ERR("EBP", "Could not find content.opf in zip");
@@ -62,7 +62,8 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
     return false;
   }
 
-  ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize, bookMetadataCache.get());
+  ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize,
+                             cacheMode == OpfCacheMode::Enabled ? bookMetadataCache.get() : nullptr);
   if (!opfParser.setup()) {
     LOG_ERR("EBP", "Could not setup content.opf parser");
     return false;
@@ -352,7 +353,7 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
         LOG_DBG("EBP", "CSS rules cache missing or stale, attempting to parse CSS files");
         cssParser->deleteCache();
 
-        if (!parseContentOpf(bookMetadataCache->coreMetadata)) {
+        if (!parseContentOpf(bookMetadataCache->coreMetadata, OpfCacheMode::Disabled)) {
           LOG_ERR("EBP", "Could not parse content.opf from cached bookMetadata for CSS files");
           // continue anyway - book will work without CSS and we'll still load any inline style CSS
         }
@@ -389,7 +390,7 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
     LOG_ERR("EBP", "Could not begin writing content.opf pass");
     return false;
   }
-  if (!parseContentOpf(bookMetadata)) {
+  if (!parseContentOpf(bookMetadata, OpfCacheMode::Enabled)) {
     LOG_ERR("EBP", "Could not parse content.opf");
     return false;
   }

lib/Epub/Epub.h

@@ -10,7 +10,7 @@
 #include "Epub/BookMetadataCache.h"
 #include "Epub/css/CssParser.h"
 
-class ZipFile;
+enum class OpfCacheMode { Disabled, Enabled };
 
 class Epub {
   // the ncx file (EPUB 2)
@@ -35,7 +35,7 @@ class Epub {
   bool syntheticTocFallbackEnabled = false;
 
   bool findContentOpfFile(std::string* contentOpfFile) const;
-  bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata);
+  bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, OpfCacheMode cacheMode);
   bool parseTocNcxFile() const;
   bool parseTocNavFile() const;
   void parseCssFiles() const;

lib/Epub/Epub/Section.cpp

@@ -12,7 +12,7 @@
 #include "parsers/ChapterHtmlSlimParser.h"
 
 namespace {
-constexpr uint8_t SECTION_FILE_VERSION = 20;
+constexpr uint8_t SECTION_FILE_VERSION = 21;
 constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) +   // SECTION_FILE_VERSION
                                  sizeof(int) +       // fontId
                                  sizeof(float) +     // lineCompression
@@ -27,6 +27,12 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) +   // SECTION_FILE_VERSION
                                  sizeof(uint32_t) +  // page LUT offset
                                  sizeof(uint32_t) +  // anchor map offset
                                  sizeof(uint32_t);   // paragraph LUT offset
+
+// On-disk paragraph LUT entry: u32 xhtmlByteOffset + u16 paragraphIndex.
+constexpr uint32_t PARAGRAPH_LUT_ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t);
+inline uint32_t paragraphLutEntryOffset(uint32_t lutStart, uint16_t page) {
+  return lutStart + page * PARAGRAPH_LUT_ENTRY_SIZE;
+}
 }  // namespace
 
 uint32_t Section::onPageComplete(std::unique_ptr<Page> page) {
@@ -309,12 +315,22 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
     serialization::writePod(file, page);
   }
 
-  // Write per-page paragraph index LUT for XPath-to-page resolution
+  // Write per-page paragraph LUT: count + array of {xhtmlByteOffset(u32), paragraphIndex(u16)}.
+  // The byte offset lets findXPathForParagraph seek near the target paragraph without scanning
+  // from the beginning of the XHTML file, reducing SD reads on large chapters.
   const uint32_t paragraphLutOffset = file.position();
-  const auto& paragraphPerPage = visitor.getParagraphIndexPerPage();
-  serialization::writePod(file, static_cast<uint16_t>(paragraphPerPage.size()));
-  for (const uint16_t& pIdx : paragraphPerPage) {
-    serialization::writePod(file, pIdx);
+  const auto& paragraphLut = visitor.getParagraphLutPerPage();
+  if (paragraphLut.size() != static_cast<size_t>(pageCount)) {
+    LOG_ERR("SCT", "Paragraph LUT size mismatch: lut=%u pageCount=%u", static_cast<uint32_t>(paragraphLut.size()),
+            static_cast<uint32_t>(pageCount));
+    file.close();
+    Storage.remove(filePath.c_str());
+    return false;
+  }
+  serialization::writePod(file, static_cast<uint16_t>(paragraphLut.size()));
+  for (const auto& entry : paragraphLut) {
+    serialization::writePod(file, entry.xhtmlByteOffset);
+    serialization::writePod(file, entry.paragraphIndex);
   }
 
   // Patch header with final pageCount, lutOffset, anchorMapOffset, and paragraphLutOffset
@@ -548,90 +564,126 @@ std::optional<uint16_t> Section::getPageForAnchor(const std::string& anchor) con
   return std::nullopt;
 }
 
+bool Section::readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32_t& outLutStart) const {
+  if (!Storage.openFileForRead("SCT", filePath, outFile)) {
+    return false;
+  }
+
+  const uint32_t fileSize = outFile.size();
+
+  outFile.seek(HEADER_SIZE - sizeof(uint32_t));
+  uint32_t paragraphLutOffset;
+  serialization::readPod(outFile, paragraphLutOffset);
+  if (fileSize < sizeof(uint16_t) || paragraphLutOffset == 0 || paragraphLutOffset > fileSize - sizeof(uint16_t)) {
+    outFile.close();
+    return false;
+  }
+
+  outFile.seek(paragraphLutOffset);
+  serialization::readPod(outFile, outCount);
+  if (outCount == 0) {
+    outFile.close();
+    return false;
+  }
+
+  const uint64_t remainingBytes = static_cast<uint64_t>(fileSize) - paragraphLutOffset;
+  const uint64_t requiredBytes = sizeof(uint16_t) + static_cast<uint64_t>(outCount) * PARAGRAPH_LUT_ENTRY_SIZE;
+  if (remainingBytes < requiredBytes) {
+    outFile.close();
+    return false;
+  }
+
+  outLutStart = paragraphLutOffset + sizeof(uint16_t);
+
+  return true;
+}
+
 std::optional<uint16_t> Section::getPageForParagraphIndex(const uint16_t pIndex) const {
   FsFile f;
-  if (!Storage.openFileForRead("SCT", filePath, f)) {
+  uint16_t count = 0;
+  uint32_t lutStart = 0;
+  if (!readParagraphLutHeader(f, count, lutStart)) {
     return std::nullopt;
   }
-
   const uint32_t fileSize = f.size();
 
-  // Read paragraph LUT offset from end of header
-  f.seek(HEADER_SIZE - sizeof(uint32_t));
-  uint32_t paragraphLutOffset;
-  serialization::readPod(f, paragraphLutOffset);
-  if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) {
-    f.close();
-    return std::nullopt;
-  }
-
-  f.seek(paragraphLutOffset);
-  uint16_t count;
-  serialization::readPod(f, count);
-  if (count == 0) {
-    f.close();
-    return std::nullopt;
-  }
-
-  // Validate that all entries fit within the file
-  const uint32_t lutEnd = paragraphLutOffset + sizeof(uint16_t) + count * sizeof(uint16_t);
-  if (lutEnd > fileSize) {
-    f.close();
-    return std::nullopt;
-  }
-
-  // Find the first page whose paragraph index >= pIndex.
-  // Each entry stores the <p> index at the time that page was completed.
-  uint16_t resultPage = count - 1;  // default to last page
+  // Each LUT entry stores the paragraph index at page-break time — i.e. the last
+  // <p> whose start tag had been seen while page i was being laid out. Paragraph
+  // P therefore first appears on the smallest i where storedPIdx[i] >= P.
   for (uint16_t i = 0; i < count; i++) {
+    const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, i) + sizeof(uint32_t);
+    const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint16_t);
+    if (requiredOffset > fileSize) {
+      f.close();
+      return std::nullopt;
+    }
+    f.seek(entryOffset);
     uint16_t pagePIdx;
     serialization::readPod(f, pagePIdx);
     if (pagePIdx >= pIndex) {
-      resultPage = i;
-      break;
+      f.close();
+      return i;
     }
   }
 
   f.close();
-  return resultPage;
+  return static_cast<uint16_t>(count - 1);
 }
 
 std::optional<uint16_t> Section::getParagraphIndexForPage(const uint16_t page) const {
   FsFile f;
-  if (!Storage.openFileForRead("SCT", filePath, f)) {
+  uint16_t count = 0;
+  uint32_t lutStart = 0;
+  if (!readParagraphLutHeader(f, count, lutStart)) {
+    return std::nullopt;
+  }
+  if (page >= count) {
+    f.close();
     return std::nullopt;
   }
 
   const uint32_t fileSize = f.size();
-
-  f.seek(HEADER_SIZE - sizeof(uint32_t));
-  uint32_t paragraphLutOffset;
-  serialization::readPod(f, paragraphLutOffset);
-  if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) {
+  const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, page) + sizeof(uint32_t);
+  const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint16_t);
+  if (requiredOffset > fileSize) {
     f.close();
     return std::nullopt;
   }
 
-  f.seek(paragraphLutOffset);
-  uint16_t count;
-  serialization::readPod(f, count);
-  if (count == 0 || page >= count) {
-    f.close();
-    return std::nullopt;
-  }
-
-  // Validate that the target entry fits within the file
-  const uint32_t entryEnd = paragraphLutOffset + sizeof(uint16_t) + (page + 1) * sizeof(uint16_t);
-  if (entryEnd > fileSize) {
-    f.close();
-    return std::nullopt;
-  }
-
-  // Seek to the entry for the requested page
-  f.seek(paragraphLutOffset + sizeof(uint16_t) + page * sizeof(uint16_t));
+  // Seek directly to the paragraphIndex field of the requested entry (skip xhtmlByteOffset)
+  f.seek(entryOffset);
   uint16_t pIdx;
   serialization::readPod(f, pIdx);
 
   f.close();
   return pIdx;
 }
+
+std::optional<uint32_t> Section::getXhtmlByteOffsetForPage(const uint16_t page) const {
+  FsFile f;
+  uint16_t count = 0;
+  uint32_t lutStart = 0;
+  if (!readParagraphLutHeader(f, count, lutStart)) {
+    return std::nullopt;
+  }
+  if (page >= count) {
+    f.close();
+    return std::nullopt;
+  }
+
+  const uint32_t fileSize = f.size();
+  const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, page);
+  const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint32_t);
+  if (requiredOffset > fileSize) {
+    f.close();
+    return std::nullopt;
+  }
+
+  f.seek(entryOffset);
+  uint32_t byteOffset;
+  serialization::readPod(f, byteOffset);
+
+  f.close();
+  // A zero offset means the entry was recorded post-parse (last page), so it's unusable as a hint.
+  return byteOffset > 0 ? std::optional<uint32_t>{byteOffset} : std::nullopt;
+}

lib/Epub/Epub/Section.h

@@ -32,6 +32,12 @@ class Section {
   void buildTocBoundaries(const std::vector<std::pair<std::string, uint16_t>>& anchors);
   void buildTocBoundariesFromFile(FsFile& f);
 
+  // Open the section file and seek to the first paragraph LUT entry, validating the header
+  // and LUT bounds against fileSize. On success, returns true with `outLutStart` set to the
+  // byte offset of the first entry (just past the count) and `outCount` to the entry count.
+  // Caller is responsible for closing `outFile`. Returns false on any I/O or validation error.
+  bool readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32_t& outLutStart) const;
+
  public:
   uint16_t pageCount = 0;
   int currentPage = 0;
@@ -68,7 +74,7 @@ class Section {
   std::optional<uint16_t> getPageForAnchor(const std::string& anchor) const;
 
   // Look up the page number for a paragraph index (1-based, from XPath p[N]).
-  // Uses the per-page paragraph index LUT stored in the section cache.
+  // Uses the per-page paragraph LUT stored in the section cache.
   // Returns nullopt if the paragraph LUT is not available (old cache format).
   std::optional<uint16_t> getPageForParagraphIndex(uint16_t pIndex) const;
 
@@ -76,4 +82,11 @@ class Section {
   // Returns the 1-based paragraph index of the last <p> element on or before the page.
   // Returns nullopt if the paragraph LUT is not available (old cache format).
   std::optional<uint16_t> getParagraphIndexForPage(uint16_t page) const;
+
+  // Look up the XHTML byte offset recorded at the page break that started the given page.
+  // This is the Expat byte position within the decompressed spine XHTML file — useful as a
+  // seek hint for findXPathForParagraph to avoid scanning from byte 0 on large chapters.
+  // Returns nullopt if the paragraph LUT is unavailable (old cache format) or offset is 0
+  // (last page, recorded after parse completion).
+  std::optional<uint32_t> getXhtmlByteOffsetForPage(uint16_t page) const;
 };

lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp

@@ -593,7 +593,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
                     (self->currentPageNextY + totalImageHeightWithSpacing > self->viewportHeight)) {
                   LOG_DBG("EHP", "Image page break: currentY=%d needed=%d viewportH=%d", self->currentPageNextY,
                           totalImageHeightWithSpacing, self->viewportHeight);
-                  self->paragraphIndexPerPage.push_back(self->xpathParagraphIndex);
+                  self->paragraphLutPerPage.push_back({self->lastBodyChildByteOffset, self->xpathParagraphIndex});
                   self->completePageFn(std::move(self->currentPage));
                   self->completedPageCount++;
                   self->currentPage.reset(new Page());
@@ -687,8 +687,16 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
   // check so that hidden <p> elements are still counted, matching ChapterXPathIndexer's
   // counting (pure XML, no CSS). This ensures paragraph indices in the section cache LUT
   // align with KOReader's crengine XPath indices.
-  if (self->xpathBodyDepth >= 0 && self->depth == self->xpathBodyDepth + 1 && strcmp(name, "p") == 0) {
-    self->xpathParagraphIndex++;
+  // At the same time, record the byte offset of every direct-body-child element start:
+  // the forward mapper's partial-parse heuristic requires the seek hint to land on a
+  // body-child boundary, otherwise partialBaseDepth can misidentify wrapped paragraphs.
+  if (self->xpathBodyDepth >= 0 && self->depth == self->xpathBodyDepth + 1) {
+    if (self->activeParser) {
+      self->lastBodyChildByteOffset = static_cast<uint32_t>(XML_GetCurrentByteIndex(self->activeParser));
+    }
+    if (strcmp(name, "p") == 0) {
+      self->xpathParagraphIndex++;
+    }
   }
 
   if (matches(name, SKIP_TAGS, NUM_SKIP_TAGS)) {
@@ -1344,6 +1352,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
   XML_SetUserData(parser, this);
   XML_SetElementHandler(parser, startElement, endElement);
   XML_SetCharacterDataHandler(parser, characterData);
+  activeParser = parser;
 
   // Compute the time taken to parse and build pages
   const uint32_t chapterStartTime = millis();
@@ -1354,6 +1363,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
       XML_StopParser(parser, XML_FALSE);                // Stop any pending processing
       XML_SetElementHandler(parser, nullptr, nullptr);  // Clear callbacks
       XML_SetCharacterDataHandler(parser, nullptr);
+      activeParser = nullptr;
       XML_ParserFree(parser);
       file.close();
       return false;
@@ -1376,6 +1386,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
       XML_StopParser(parser, XML_FALSE);                // Stop any pending processing
       XML_SetElementHandler(parser, nullptr, nullptr);  // Clear callbacks
       XML_SetCharacterDataHandler(parser, nullptr);
+      activeParser = nullptr;
       XML_ParserFree(parser);
       file.close();
       return false;
@@ -1389,6 +1400,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
       XML_StopParser(parser, XML_FALSE);                // Stop any pending processing
       XML_SetElementHandler(parser, nullptr, nullptr);  // Clear callbacks
       XML_SetCharacterDataHandler(parser, nullptr);
+      activeParser = nullptr;
       XML_ParserFree(parser);
       file.close();
       return false;
@@ -1400,6 +1412,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
   XML_StopParser(parser, XML_FALSE);                // Stop any pending processing
   XML_SetElementHandler(parser, nullptr, nullptr);  // Clear callbacks
   XML_SetCharacterDataHandler(parser, nullptr);
+  activeParser = nullptr;
   XML_ParserFree(parser);
   file.close();
 
@@ -1410,7 +1423,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
       anchorData.push_back({std::move(pendingAnchorId), static_cast<uint16_t>(completedPageCount)});
       pendingAnchorId.clear();
     }
-    paragraphIndexPerPage.push_back(xpathParagraphIndex);
+    paragraphLutPerPage.push_back({0u, xpathParagraphIndex});  // post-parse: no byte offset available
     completePageFn(std::move(currentPage));
     completedPageCount++;
     currentPage.reset();
@@ -1431,7 +1444,7 @@ ParsedText::LineProcessResult ChapterHtmlSlimParser::addLineToPage(std::shared_p
   }
 
   if (currentPageNextY + lineHeight > viewportHeight) {
-    paragraphIndexPerPage.push_back(xpathParagraphIndex);
+    paragraphLutPerPage.push_back({lastBodyChildByteOffset, xpathParagraphIndex});
     completePageFn(std::move(currentPage));
     completedPageCount++;
     currentPage.reset(new Page());

lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h

@@ -90,9 +90,25 @@ class ChapterHtmlSlimParser {
   // Counts <p> sibling indices (1-based, matching XPath convention) during page building.
   // Stored per page in the section cache so that XPath p[N] can be resolved to a page
   // without reparsing, and current page can generate an XPath without reparsing.
-  uint16_t xpathParagraphIndex = 0;             // current <p> sibling index (1-based)
-  int xpathBodyDepth = -1;                      // depth of the <body> element (-1 = not yet seen)
-  std::vector<uint16_t> paragraphIndexPerPage;  // <p> index at each page completion
+  uint16_t xpathParagraphIndex = 0;  // current <p> sibling index (1-based)
+  int xpathBodyDepth = -1;           // depth of the <body> element (-1 = not yet seen)
+  // Byte offset of the most recent direct-body-child element start (any tag at xpathBodyDepth+1).
+  // Recorded at the same depth condition that increments xpathParagraphIndex, so the stored
+  // offset is guaranteed to land on a body-child element boundary. This keeps the XPath forward
+  // mapper's partial-parse heuristic reliable for wrapped chapters: without this, the offset
+  // could point mid-way into a nested <div>/<section>, which confuses partialBaseDepth.
+  uint32_t lastBodyChildByteOffset = 0;
+
+  struct ParagraphLutEntry {
+    uint32_t xhtmlByteOffset;  // byte offset of most recent body-child element start at page break
+    uint16_t paragraphIndex;   // 1-based <p> index at page completion
+  };
+  std::vector<ParagraphLutEntry> paragraphLutPerPage;  // deep LUT: one entry per page
+
+  // Active parser handle during parseAndBuildPages(), nullptr otherwise.
+  // Stored as a member so page-break sites (addLineToPage, image breaks) can call
+  // XML_GetCurrentByteIndex without needing the parser threaded through every call.
+  XML_Parser activeParser = nullptr;
 
   // Footnote link tracking
   bool insideFootnoteLink = false;
@@ -154,5 +170,5 @@ class ChapterHtmlSlimParser {
   ParsedText::LineProcessResult addLineToPage(std::shared_ptr<TextBlock> line, bool lineEndsWithHyphenatedWord,
                                               bool suppressHyphenationRetry);
   const std::vector<std::pair<std::string, uint16_t>>& getAnchors() const { return anchorData; }
-  const std::vector<uint16_t>& getParagraphIndexPerPage() const { return paragraphIndexPerPage; }
+  const std::vector<ParagraphLutEntry>& getParagraphLutPerPage() const { return paragraphLutPerPage; }
 };