Merge pull request #109 from jpirnay/kosync_extension

refactor: Picking up some itsthisjustin ideas for KOreader synchronisation
This commit is contained in:
jpirnay
2026-04-20 14:21:31 +02:00
committed by GitHub
24 changed files with 493 additions and 140 deletions
+11 -8
View File
@@ -104,7 +104,7 @@ if (parsedSize != fileSize) {
## `section.bin`
### Version 20
### Version 21
ImHex Pattern:
@@ -114,7 +114,7 @@ import std.string;
import std.core;
// === Configuration ===
#define EXPECTED_VERSION 20
#define EXPECTED_VERSION 21
#define MAX_STRING_LENGTH 65535
// === String Structure ===
@@ -206,7 +206,7 @@ struct SectionBin {
u8 imageRendering;
u32 pageLutOffset [[comment("Offset to page offset LUT")]];
u32 anchorMapOffset [[comment("Offset to anchor map")]];
u32 paragraphLutOffset [[comment("Offset to per-page paragraph index LUT")]];
u32 paragraphLutOffset [[comment("Offset to per-page paragraph LUT (byte offset + <p> index)")]];
Page page[pageCount];
@@ -223,12 +223,15 @@ struct SectionBin {
u16 anchorCount;
AnchorEntry anchors[anchorCount];
// === Paragraph Index LUT ===
// One entry per page: the 1-based <p> sibling index (XPath convention)
// at the time each page was completed during parsing.
// Used to resolve KOReader XPath p[N] positions to page numbers.
// === Paragraph LUT (deep entries) ===
// One entry per page: XHTML byte offset at the page break + 1-based <p> sibling index.
// xhtmlByteOffset is the Expat byte position within the decompressed spine XHTML at the
// moment the page break fired — used as a seek hint to avoid scanning from byte 0 when
// generating XPaths for upload. 0 means no hint (last page, recorded post-parse).
// paragraphIndex is 1-based, matching KOReader XPath p[N] convention.
struct ParagraphLutEntry { u32 xhtmlByteOffset; u16 paragraphIndex; };
u16 paragraphEntryCount;
u16 paragraphIndexPerPage[paragraphEntryCount] [[comment("1-based <p> index at page completion")]];
ParagraphLutEntry paragraphLut[paragraphEntryCount];
};
// === File Parsing ===
+5 -4
View File
@@ -45,7 +45,7 @@ bool Epub::findContentOpfFile(std::string* contentOpfFile) const {
return true;
}
bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, OpfCacheMode cacheMode) {
std::string contentOpfFilePath;
if (!findContentOpfFile(&contentOpfFilePath)) {
LOG_ERR("EBP", "Could not find content.opf in zip");
@@ -62,7 +62,8 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
return false;
}
ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize, bookMetadataCache.get());
ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize,
cacheMode == OpfCacheMode::Enabled ? bookMetadataCache.get() : nullptr);
if (!opfParser.setup()) {
LOG_ERR("EBP", "Could not setup content.opf parser");
return false;
@@ -352,7 +353,7 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
LOG_DBG("EBP", "CSS rules cache missing or stale, attempting to parse CSS files");
cssParser->deleteCache();
if (!parseContentOpf(bookMetadataCache->coreMetadata)) {
if (!parseContentOpf(bookMetadataCache->coreMetadata, OpfCacheMode::Disabled)) {
LOG_ERR("EBP", "Could not parse content.opf from cached bookMetadata for CSS files");
// continue anyway - book will work without CSS and we'll still load any inline style CSS
}
@@ -389,7 +390,7 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
LOG_ERR("EBP", "Could not begin writing content.opf pass");
return false;
}
if (!parseContentOpf(bookMetadata)) {
if (!parseContentOpf(bookMetadata, OpfCacheMode::Enabled)) {
LOG_ERR("EBP", "Could not parse content.opf");
return false;
}
+2 -2
View File
@@ -10,7 +10,7 @@
#include "Epub/BookMetadataCache.h"
#include "Epub/css/CssParser.h"
class ZipFile;
enum class OpfCacheMode { Disabled, Enabled };
class Epub {
// the ncx file (EPUB 2)
@@ -35,7 +35,7 @@ class Epub {
bool syntheticTocFallbackEnabled = false;
bool findContentOpfFile(std::string* contentOpfFile) const;
bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata);
bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, OpfCacheMode cacheMode);
bool parseTocNcxFile() const;
bool parseTocNavFile() const;
void parseCssFiles() const;
+113 -61
View File
@@ -12,7 +12,7 @@
#include "parsers/ChapterHtmlSlimParser.h"
namespace {
constexpr uint8_t SECTION_FILE_VERSION = 20;
constexpr uint8_t SECTION_FILE_VERSION = 21;
constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(int) + // fontId
sizeof(float) + // lineCompression
@@ -27,6 +27,12 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(uint32_t) + // page LUT offset
sizeof(uint32_t) + // anchor map offset
sizeof(uint32_t); // paragraph LUT offset
// On-disk paragraph LUT entry: u32 xhtmlByteOffset + u16 paragraphIndex.
constexpr uint32_t PARAGRAPH_LUT_ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t);
inline uint32_t paragraphLutEntryOffset(uint32_t lutStart, uint16_t page) {
return lutStart + page * PARAGRAPH_LUT_ENTRY_SIZE;
}
} // namespace
uint32_t Section::onPageComplete(std::unique_ptr<Page> page) {
@@ -309,12 +315,22 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
serialization::writePod(file, page);
}
// Write per-page paragraph index LUT for XPath-to-page resolution
// Write per-page paragraph LUT: count + array of {xhtmlByteOffset(u32), paragraphIndex(u16)}.
// The byte offset lets findXPathForParagraph seek near the target paragraph without scanning
// from the beginning of the XHTML file, reducing SD reads on large chapters.
const uint32_t paragraphLutOffset = file.position();
const auto& paragraphPerPage = visitor.getParagraphIndexPerPage();
serialization::writePod(file, static_cast<uint16_t>(paragraphPerPage.size()));
for (const uint16_t& pIdx : paragraphPerPage) {
serialization::writePod(file, pIdx);
const auto& paragraphLut = visitor.getParagraphLutPerPage();
if (paragraphLut.size() != static_cast<size_t>(pageCount)) {
LOG_ERR("SCT", "Paragraph LUT size mismatch: lut=%u pageCount=%u", static_cast<uint32_t>(paragraphLut.size()),
static_cast<uint32_t>(pageCount));
file.close();
Storage.remove(filePath.c_str());
return false;
}
serialization::writePod(file, static_cast<uint16_t>(paragraphLut.size()));
for (const auto& entry : paragraphLut) {
serialization::writePod(file, entry.xhtmlByteOffset);
serialization::writePod(file, entry.paragraphIndex);
}
// Patch header with final pageCount, lutOffset, anchorMapOffset, and paragraphLutOffset
@@ -548,90 +564,126 @@ std::optional<uint16_t> Section::getPageForAnchor(const std::string& anchor) con
return std::nullopt;
}
bool Section::readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32_t& outLutStart) const {
if (!Storage.openFileForRead("SCT", filePath, outFile)) {
return false;
}
const uint32_t fileSize = outFile.size();
outFile.seek(HEADER_SIZE - sizeof(uint32_t));
uint32_t paragraphLutOffset;
serialization::readPod(outFile, paragraphLutOffset);
if (fileSize < sizeof(uint16_t) || paragraphLutOffset == 0 || paragraphLutOffset > fileSize - sizeof(uint16_t)) {
outFile.close();
return false;
}
outFile.seek(paragraphLutOffset);
serialization::readPod(outFile, outCount);
if (outCount == 0) {
outFile.close();
return false;
}
const uint64_t remainingBytes = static_cast<uint64_t>(fileSize) - paragraphLutOffset;
const uint64_t requiredBytes = sizeof(uint16_t) + static_cast<uint64_t>(outCount) * PARAGRAPH_LUT_ENTRY_SIZE;
if (remainingBytes < requiredBytes) {
outFile.close();
return false;
}
outLutStart = paragraphLutOffset + sizeof(uint16_t);
return true;
}
std::optional<uint16_t> Section::getPageForParagraphIndex(const uint16_t pIndex) const {
FsFile f;
if (!Storage.openFileForRead("SCT", filePath, f)) {
uint16_t count = 0;
uint32_t lutStart = 0;
if (!readParagraphLutHeader(f, count, lutStart)) {
return std::nullopt;
}
const uint32_t fileSize = f.size();
// Read paragraph LUT offset from end of header
f.seek(HEADER_SIZE - sizeof(uint32_t));
uint32_t paragraphLutOffset;
serialization::readPod(f, paragraphLutOffset);
if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) {
f.close();
return std::nullopt;
}
f.seek(paragraphLutOffset);
uint16_t count;
serialization::readPod(f, count);
if (count == 0) {
f.close();
return std::nullopt;
}
// Validate that all entries fit within the file
const uint32_t lutEnd = paragraphLutOffset + sizeof(uint16_t) + count * sizeof(uint16_t);
if (lutEnd > fileSize) {
f.close();
return std::nullopt;
}
// Find the first page whose paragraph index >= pIndex.
// Each entry stores the <p> index at the time that page was completed.
uint16_t resultPage = count - 1; // default to last page
// Each LUT entry stores the paragraph index at page-break time — i.e. the last
// <p> whose start tag had been seen while page i was being laid out. Paragraph
// P therefore first appears on the smallest i where storedPIdx[i] >= P.
for (uint16_t i = 0; i < count; i++) {
const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, i) + sizeof(uint32_t);
const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint16_t);
if (requiredOffset > fileSize) {
f.close();
return std::nullopt;
}
f.seek(entryOffset);
uint16_t pagePIdx;
serialization::readPod(f, pagePIdx);
if (pagePIdx >= pIndex) {
resultPage = i;
break;
f.close();
return i;
}
}
f.close();
return resultPage;
return static_cast<uint16_t>(count - 1);
}
std::optional<uint16_t> Section::getParagraphIndexForPage(const uint16_t page) const {
FsFile f;
if (!Storage.openFileForRead("SCT", filePath, f)) {
uint16_t count = 0;
uint32_t lutStart = 0;
if (!readParagraphLutHeader(f, count, lutStart)) {
return std::nullopt;
}
if (page >= count) {
f.close();
return std::nullopt;
}
const uint32_t fileSize = f.size();
f.seek(HEADER_SIZE - sizeof(uint32_t));
uint32_t paragraphLutOffset;
serialization::readPod(f, paragraphLutOffset);
if (paragraphLutOffset == 0 || paragraphLutOffset >= fileSize) {
const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, page) + sizeof(uint32_t);
const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint16_t);
if (requiredOffset > fileSize) {
f.close();
return std::nullopt;
}
f.seek(paragraphLutOffset);
uint16_t count;
serialization::readPod(f, count);
if (count == 0 || page >= count) {
f.close();
return std::nullopt;
}
// Validate that the target entry fits within the file
const uint32_t entryEnd = paragraphLutOffset + sizeof(uint16_t) + (page + 1) * sizeof(uint16_t);
if (entryEnd > fileSize) {
f.close();
return std::nullopt;
}
// Seek to the entry for the requested page
f.seek(paragraphLutOffset + sizeof(uint16_t) + page * sizeof(uint16_t));
// Seek directly to the paragraphIndex field of the requested entry (skip xhtmlByteOffset)
f.seek(entryOffset);
uint16_t pIdx;
serialization::readPod(f, pIdx);
f.close();
return pIdx;
}
std::optional<uint32_t> Section::getXhtmlByteOffsetForPage(const uint16_t page) const {
FsFile f;
uint16_t count = 0;
uint32_t lutStart = 0;
if (!readParagraphLutHeader(f, count, lutStart)) {
return std::nullopt;
}
if (page >= count) {
f.close();
return std::nullopt;
}
const uint32_t fileSize = f.size();
const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, page);
const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint32_t);
if (requiredOffset > fileSize) {
f.close();
return std::nullopt;
}
f.seek(entryOffset);
uint32_t byteOffset;
serialization::readPod(f, byteOffset);
f.close();
// A zero offset means the entry was recorded post-parse (last page), so it's unusable as a hint.
return byteOffset > 0 ? std::optional<uint32_t>{byteOffset} : std::nullopt;
}
+14 -1
View File
@@ -32,6 +32,12 @@ class Section {
void buildTocBoundaries(const std::vector<std::pair<std::string, uint16_t>>& anchors);
void buildTocBoundariesFromFile(FsFile& f);
// Open the section file and seek to the first paragraph LUT entry, validating the header
// and LUT bounds against fileSize. On success, returns true with `outLutStart` set to the
// byte offset of the first entry (just past the count) and `outCount` to the entry count.
// Caller is responsible for closing `outFile`. Returns false on any I/O or validation error.
bool readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32_t& outLutStart) const;
public:
uint16_t pageCount = 0;
int currentPage = 0;
@@ -68,7 +74,7 @@ class Section {
std::optional<uint16_t> getPageForAnchor(const std::string& anchor) const;
// Look up the page number for a paragraph index (1-based, from XPath p[N]).
// Uses the per-page paragraph index LUT stored in the section cache.
// Uses the per-page paragraph LUT stored in the section cache.
// Returns nullopt if the paragraph LUT is not available (old cache format).
std::optional<uint16_t> getPageForParagraphIndex(uint16_t pIndex) const;
@@ -76,4 +82,11 @@ class Section {
// Returns the 1-based paragraph index of the last <p> element on or before the page.
// Returns nullopt if the paragraph LUT is not available (old cache format).
std::optional<uint16_t> getParagraphIndexForPage(uint16_t page) const;
// Look up the XHTML byte offset recorded at the page break that started the given page.
// This is the Expat byte position within the decompressed spine XHTML file — useful as a
// seek hint for findXPathForParagraph to avoid scanning from byte 0 on large chapters.
// Returns nullopt if the paragraph LUT is unavailable (old cache format) or offset is 0
// (last page, recorded after parse completion).
std::optional<uint32_t> getXhtmlByteOffsetForPage(uint16_t page) const;
};
@@ -593,7 +593,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
(self->currentPageNextY + totalImageHeightWithSpacing > self->viewportHeight)) {
LOG_DBG("EHP", "Image page break: currentY=%d needed=%d viewportH=%d", self->currentPageNextY,
totalImageHeightWithSpacing, self->viewportHeight);
self->paragraphIndexPerPage.push_back(self->xpathParagraphIndex);
self->paragraphLutPerPage.push_back({self->lastBodyChildByteOffset, self->xpathParagraphIndex});
self->completePageFn(std::move(self->currentPage));
self->completedPageCount++;
self->currentPage.reset(new Page());
@@ -687,8 +687,16 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// check so that hidden <p> elements are still counted, matching ChapterXPathIndexer's
// counting (pure XML, no CSS). This ensures paragraph indices in the section cache LUT
// align with KOReader's crengine XPath indices.
if (self->xpathBodyDepth >= 0 && self->depth == self->xpathBodyDepth + 1 && strcmp(name, "p") == 0) {
self->xpathParagraphIndex++;
// At the same time, record the byte offset of every direct-body-child element start:
// the forward mapper's partial-parse heuristic requires the seek hint to land on a
// body-child boundary, otherwise partialBaseDepth can misidentify wrapped paragraphs.
if (self->xpathBodyDepth >= 0 && self->depth == self->xpathBodyDepth + 1) {
if (self->activeParser) {
self->lastBodyChildByteOffset = static_cast<uint32_t>(XML_GetCurrentByteIndex(self->activeParser));
}
if (strcmp(name, "p") == 0) {
self->xpathParagraphIndex++;
}
}
if (matches(name, SKIP_TAGS, NUM_SKIP_TAGS)) {
@@ -1344,6 +1352,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
XML_SetUserData(parser, this);
XML_SetElementHandler(parser, startElement, endElement);
XML_SetCharacterDataHandler(parser, characterData);
activeParser = parser;
// Compute the time taken to parse and build pages
const uint32_t chapterStartTime = millis();
@@ -1354,6 +1363,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
XML_StopParser(parser, XML_FALSE); // Stop any pending processing
XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks
XML_SetCharacterDataHandler(parser, nullptr);
activeParser = nullptr;
XML_ParserFree(parser);
file.close();
return false;
@@ -1376,6 +1386,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
XML_StopParser(parser, XML_FALSE); // Stop any pending processing
XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks
XML_SetCharacterDataHandler(parser, nullptr);
activeParser = nullptr;
XML_ParserFree(parser);
file.close();
return false;
@@ -1389,6 +1400,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
XML_StopParser(parser, XML_FALSE); // Stop any pending processing
XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks
XML_SetCharacterDataHandler(parser, nullptr);
activeParser = nullptr;
XML_ParserFree(parser);
file.close();
return false;
@@ -1400,6 +1412,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
XML_StopParser(parser, XML_FALSE); // Stop any pending processing
XML_SetElementHandler(parser, nullptr, nullptr); // Clear callbacks
XML_SetCharacterDataHandler(parser, nullptr);
activeParser = nullptr;
XML_ParserFree(parser);
file.close();
@@ -1410,7 +1423,7 @@ bool ChapterHtmlSlimParser::parseAndBuildPages() {
anchorData.push_back({std::move(pendingAnchorId), static_cast<uint16_t>(completedPageCount)});
pendingAnchorId.clear();
}
paragraphIndexPerPage.push_back(xpathParagraphIndex);
paragraphLutPerPage.push_back({0u, xpathParagraphIndex}); // post-parse: no byte offset available
completePageFn(std::move(currentPage));
completedPageCount++;
currentPage.reset();
@@ -1431,7 +1444,7 @@ ParsedText::LineProcessResult ChapterHtmlSlimParser::addLineToPage(std::shared_p
}
if (currentPageNextY + lineHeight > viewportHeight) {
paragraphIndexPerPage.push_back(xpathParagraphIndex);
paragraphLutPerPage.push_back({lastBodyChildByteOffset, xpathParagraphIndex});
completePageFn(std::move(currentPage));
completedPageCount++;
currentPage.reset(new Page());
+20 -4
View File
@@ -90,9 +90,25 @@ class ChapterHtmlSlimParser {
// Counts <p> sibling indices (1-based, matching XPath convention) during page building.
// Stored per page in the section cache so that XPath p[N] can be resolved to a page
// without reparsing, and current page can generate an XPath without reparsing.
uint16_t xpathParagraphIndex = 0; // current <p> sibling index (1-based)
int xpathBodyDepth = -1; // depth of the <body> element (-1 = not yet seen)
std::vector<uint16_t> paragraphIndexPerPage; // <p> index at each page completion
uint16_t xpathParagraphIndex = 0; // current <p> sibling index (1-based)
int xpathBodyDepth = -1; // depth of the <body> element (-1 = not yet seen)
// Byte offset of the most recent direct-body-child element start (any tag at xpathBodyDepth+1).
// Recorded at the same depth condition that increments xpathParagraphIndex, so the stored
// offset is guaranteed to land on a body-child element boundary. This keeps the XPath forward
// mapper's partial-parse heuristic reliable for wrapped chapters: without this, the offset
// could point mid-way into a nested <div>/<section>, which confuses partialBaseDepth.
uint32_t lastBodyChildByteOffset = 0;
struct ParagraphLutEntry {
uint32_t xhtmlByteOffset; // byte offset of most recent body-child element start at page break
uint16_t paragraphIndex; // 1-based <p> index at page completion
};
std::vector<ParagraphLutEntry> paragraphLutPerPage; // deep LUT: one entry per page
// Active parser handle during parseAndBuildPages(), nullptr otherwise.
// Stored as a member so page-break sites (addLineToPage, image breaks) can call
// XML_GetCurrentByteIndex without needing the parser threaded through every call.
XML_Parser activeParser = nullptr;
// Footnote link tracking
bool insideFootnoteLink = false;
@@ -154,5 +170,5 @@ class ChapterHtmlSlimParser {
ParsedText::LineProcessResult addLineToPage(std::shared_ptr<TextBlock> line, bool lineEndsWithHyphenatedWord,
bool suppressHyphenationRetry);
const std::vector<std::pair<std::string, uint16_t>>& getAnchors() const { return anchorData; }
const std::vector<uint16_t>& getParagraphIndexPerPage() const { return paragraphIndexPerPage; }
const std::vector<ParagraphLutEntry>& getParagraphLutPerPage() const { return paragraphLutPerPage; }
};
@@ -124,6 +124,127 @@ size_t getTotalTextBytesCached(const std::shared_ptr<Epub>& epub, const int spin
} // namespace
// Paragraph-targeted forward mapper.
// Counts direct-body-child <p> elements (matching ChapterHtmlSlimParser's xpathBodyDepth guard)
// and stops at the Nth one, emitting its full-ancestry XPath. The seek hint avoids scanning
// from byte 0 when the section LUT has a byte offset for a nearby page break.
namespace {
struct ParagraphState : StackState {
int spineIndex;
uint16_t targetParagraph; // 1-based
uint16_t paragraphCount = 0;
std::string result;
XML_Parser parser = nullptr;
// When parsing from a seek offset, the DOM context (html/body ancestors) is missing from
// the parser's perspective. partialParse=true relaxes the bodyIdx() check and instead
// counts any <p> at depth 0 relative to the first element seen (a heuristic that works
// because we know we're already inside <body> in the source document).
bool partialParse = false;
int partialBaseDepth = -1; // stack depth when the first element is seen in partial mode
ParagraphState(const int spineIndex, const uint16_t targetParagraph, const uint16_t startParagraphCount,
const bool partialParse)
: spineIndex(spineIndex),
targetParagraph(targetParagraph),
paragraphCount(startParagraphCount),
partialParse(partialParse) {}
};
void XMLCALL paragraphStartCb(void* ud, const XML_Char* rawName, const XML_Char**) {
auto* s = static_cast<ParagraphState*>(ud);
s->pushElement(rawName);
if (!s->result.empty() || s->stack.empty() || s->stack.back().tag != "p") {
return;
}
bool isDirectBodyChild = false;
if (s->partialParse) {
// In partial mode the DOM context (html/body ancestors) is absent from the parser.
// We record the stack depth of the first element encountered as the body-equivalent
// depth; direct body children are one level deeper. This only works for flat EPUBs
// where paragraphs are direct children of <body> — for wrapped chapters the partial
// parse will find nothing and the caller retries from byte 0 with full context.
if (s->partialBaseDepth < 0) {
s->partialBaseDepth = static_cast<int>(s->stack.size()) - 1;
}
isDirectBodyChild = (static_cast<int>(s->stack.size()) - 1 == s->partialBaseDepth);
} else {
const int bi = s->bodyIdx();
isDirectBodyChild = (bi >= 0 && static_cast<int>(s->stack.size()) == bi + 2);
}
if (isDirectBodyChild) {
s->paragraphCount++;
if (s->paragraphCount >= s->targetParagraph) {
s->result = s->currentXPath(s->spineIndex);
if (s->parser) {
XML_StopParser(s->parser, XML_FALSE);
}
}
}
}
void XMLCALL paragraphEndCb(void* ud, const XML_Char*) { static_cast<ParagraphState*>(ud)->popElement(); }
} // namespace
std::string findXPathForParagraphInternal(const std::shared_ptr<Epub>& epub, const int spineIndex,
const uint16_t paragraphIndex, const uint32_t seekHint,
const uint16_t startParagraphCount) {
if (!epub || paragraphIndex == 0) {
return "";
}
const std::string tmpPath = decompressToTempFile(epub, spineIndex);
if (tmpPath.empty()) {
return "";
}
const bool partialParse = seekHint > 0;
ParagraphState state(spineIndex, paragraphIndex, partialParse ? startParagraphCount : 0, partialParse);
XML_Parser parser = XML_ParserCreate(nullptr);
if (!parser) {
Storage.remove(tmpPath.c_str());
return "";
}
state.parser = parser;
XML_SetUserData(parser, &state);
XML_SetElementHandler(parser, paragraphStartCb, paragraphEndCb);
// No character data handler needed — we only care about element structure.
XML_SetDefaultHandlerExpand(parser, parserDefaultCb<ParagraphState>);
// Use seek hint from section LUT if available — avoids scanning the whole chapter.
// If the partial parse misses the target (e.g. the hint overshot), retry from byte 0.
runParseFromOffset(parser, tmpPath, seekHint);
if (state.result.empty() && seekHint > 0) {
// Partial parse missed — reset and retry from beginning with full-document context.
XML_ParserFree(parser);
parser = XML_ParserCreate(nullptr);
if (!parser) {
LOG_ERR("KOX", "XML_ParserCreate failed on retry: spine=%d p[%u] tmp=%s", spineIndex, paragraphIndex,
tmpPath.c_str());
} else {
ParagraphState fullState(spineIndex, paragraphIndex, 0, false);
fullState.parser = parser;
XML_SetUserData(parser, &fullState);
XML_SetElementHandler(parser, paragraphStartCb, paragraphEndCb);
XML_SetDefaultHandlerExpand(parser, parserDefaultCb<ParagraphState>);
runParse(parser, tmpPath);
state.result = fullState.result;
}
}
XML_ParserFree(parser);
Storage.remove(tmpPath.c_str());
LOG_DBG("KOX", "Paragraph: spine=%d p[%u] seekHint=%u -> %s", spineIndex, paragraphIndex, seekHint,
state.result.empty() ? "(not found)" : state.result.c_str());
return state.result;
}
std::string findXPathForProgressInternal(const std::shared_ptr<Epub>& epub, const int spineIndex,
const float intraSpineProgress) {
const std::string tmpPath = decompressToTempFile(epub, spineIndex);
@@ -9,4 +9,13 @@ namespace ChapterXPathIndexerInternal {
std::string findXPathForProgressInternal(const std::shared_ptr<Epub>& epub, int spineIndex, float intraSpineProgress);
// Find the full-ancestry XPath for the paragraphIndex-th direct-body-child <p> element.
// paragraphIndex is 1-based, matching the section paragraph LUT and KOReader XPath convention.
// seekHint is an optional XHTML byte offset to start scanning from (0 = scan from beginning).
// startParagraphCount is the number of body-child <p> elements already seen before seekHint
// (i.e. the paragraphIndex of the LUT entry at the seek page, minus 1). Ignored when seekHint=0.
// Returns empty string on failure; caller should fall back to findXPathForProgressInternal.
std::string findXPathForParagraphInternal(const std::shared_ptr<Epub>& epub, int spineIndex, uint16_t paragraphIndex,
uint32_t seekHint = 0, uint16_t startParagraphCount = 0);
} // namespace ChapterXPathIndexerInternal
+23 -4
View File
@@ -21,6 +21,12 @@ std::string ChapterXPathIndexer::findXPathForProgress(const std::shared_ptr<Epub
return findXPathForProgressInternal(epub, spineIndex, intraSpineProgress);
}
std::string ChapterXPathIndexer::findXPathForParagraph(const std::shared_ptr<Epub>& epub, const int spineIndex,
const uint16_t paragraphIndex, const uint32_t seekHint,
const uint16_t startParagraphCount) {
return findXPathForParagraphInternal(epub, spineIndex, paragraphIndex, seekHint, startParagraphCount);
}
bool ChapterXPathIndexer::findProgressForXPath(const std::shared_ptr<Epub>& epub, const int spineIndex,
const std::string& xpath, float& outIntraSpineProgress,
bool& outExactMatch) {
@@ -83,10 +89,23 @@ bool ChapterXPathIndexer::tryExtractParagraphIndexFromXPath(const std::string& x
return false;
}
// Only accept p[...] that is a direct child of the /body segment — reject
// paths with intermediate ancestor segments (e.g. /body/.../div[4]/p[1])
// which would collapse structurally different locations to the same index.
const size_t bodyEnd = (secondBody != std::string::npos ? secondBody : 0) + bodyKey.size();
// Only accept p[...] that is a direct child of the /body segment.
// The section paragraph LUT counts only direct-body-child <p> elements (matching
// KOReader's crengine pure-XML counting). A nested path like /body/div[2]/p[4]
// cannot be mapped to our flat LUT index — the p[4] there is the 4th sibling inside
// div[2], not the 4th <p> child of <body>. Deeply-nested XPaths fall through to
// ChapterXPathIndexer::findProgressForXPath which handles full-ancestry matching.
size_t bodyEnd = (secondBody != std::string::npos ? secondBody : 0) + bodyKey.size();
if (bodyEnd < normalized.size() && normalized[bodyEnd] == '[') {
const size_t idxStart = bodyEnd + 1;
size_t idxEnd = idxStart;
while (idxEnd < normalized.size() && std::isdigit(static_cast<unsigned char>(normalized[idxEnd]))) {
idxEnd++;
}
if (idxEnd < normalized.size() && normalized[idxEnd] == ']') {
bodyEnd = idxEnd + 1;
}
}
if (pos != bodyEnd) {
return false;
}
+22
View File
@@ -65,6 +65,28 @@ class ChapterXPathIndexer {
*/
static bool tryExtractSpineIndexFromXPath(const std::string& xpath, int& outSpineIndex);
/**
* Find the full-ancestry XPath for the Nth direct-body-child <p> element.
*
* Counts only <p> elements that are direct children of <body>, matching the semantics
* of the section paragraph LUT built by ChapterHtmlSlimParser.
*
* @param epub Loaded EPUB instance
* @param spineIndex Spine item index to parse
* @param paragraphIndex 1-based paragraph index (from section LUT or XPath p[N])
* @param seekHint Optional XHTML byte offset to start scanning from (0 = from beginning).
* Pass Section::getXhtmlByteOffsetForPage() to avoid scanning the whole file.
* @param startParagraphCount Optional seed count (default 0) of direct-body-child <p> elements
* that precede the seekHint position. Should be provided when seekHint > 0 to
* avoid counting from scratch mid-document; callers should pass the paragraph
* index of the LUT entry at the seek page minus 1. If the partial parse with
* this seed doesn't find the target, the function falls back to runParse from
* byte 0 and re-counts with startParagraphCount = 0.
* @return Full-ancestry XPath like "/body/DocFragment[N]/body/div[1]/p[3]", or empty on failure
*/
static std::string findXPathForParagraph(const std::shared_ptr<Epub>& epub, int spineIndex, uint16_t paragraphIndex,
uint32_t seekHint = 0, uint16_t startParagraphCount = 0);
/**
* Extract the paragraph index from a KOReader XPath.
* Looks for the first /p[N] segment after /body/ and returns N (1-based).
@@ -245,29 +245,59 @@ std::string decompressToTempFile(const std::shared_ptr<Epub>& epub, const int sp
return tmpPath;
}
namespace {
// Pump the open `file` through `parser` in fixed-size chunks. Returns true on clean EOF or
// XML_ERROR_ABORTED (caller used XML_StopParser to signal an early success). Returns false on
// XML_GetBuffer failure or any other parse error. The file is left open — caller closes it.
bool pumpExpatFromFile(XML_Parser parser, FsFile& file) {
constexpr size_t kBufSize = 1024;
int done;
do {
void* const buf = XML_GetBuffer(parser, kBufSize);
if (!buf) {
return false;
}
const size_t len = file.read(buf, kBufSize);
done = file.available() == 0;
if (XML_ParseBuffer(parser, static_cast<int>(len), done) == XML_STATUS_ERROR) {
return XML_GetErrorCode(parser) == XML_ERROR_ABORTED;
}
} while (!done);
return true;
}
} // namespace
bool runParse(XML_Parser parser, const std::string& path) {
FsFile file;
if (!Storage.openFileForRead("KOX", path, file)) {
return false;
}
const bool ok = pumpExpatFromFile(parser, file);
file.close();
return ok;
}
constexpr size_t kBufSize = 1024;
bool ok = true;
int done;
do {
void* const buf = XML_GetBuffer(parser, kBufSize);
if (!buf) {
ok = false;
break;
}
const size_t len = file.read(buf, kBufSize);
done = file.available() == 0;
if (XML_ParseBuffer(parser, static_cast<int>(len), done) == XML_STATUS_ERROR) {
ok = (XML_GetErrorCode(parser) == XML_ERROR_ABORTED);
break;
}
} while (!done);
// Starts Expat mid-document. Since the parser has no ancestor context (html/body stack is
// missing), unmatched closing tags may appear, and callbacks emitted before the first start
// tag can look structurally odd — an empty result is normal here. Callers
// (ChapterXPathForwardMapper.cpp) recognise the empty result and fall back to runParse from
// byte 0 with full document context.
bool runParseFromOffset(XML_Parser parser, const std::string& path, const uint32_t seekBytes) {
if (seekBytes == 0) {
return runParse(parser, path);
}
FsFile file;
if (!Storage.openFileForRead("KOX", path, file)) {
return false;
}
if (!file.seek(seekBytes)) {
file.close();
return runParse(parser, path); // fall back to full scan if seek fails
}
const bool ok = pumpExpatFromFile(parser, file);
file.close();
return ok;
}
@@ -25,6 +25,10 @@ bool isAncestorPath(const std::string& prefix, const std::string& path);
std::string decompressToTempFile(const std::shared_ptr<Epub>& epub, int spineIndex);
bool runParse(XML_Parser parser, const std::string& path);
// Like runParse but skips the first seekBytes bytes before feeding data to the parser.
// Valid only when the parser is freshly created and the seek position is known to be on an XML
// boundary (e.g. the Expat byte offset recorded at a page break).
bool runParseFromOffset(XML_Parser parser, const std::string& path, uint32_t seekBytes);
bool isEntityRef(const XML_Char* text, int len);
size_t countTotalTextBytes(const std::string& tmpPath);
@@ -54,6 +54,8 @@ struct StackState {
}
}
void onCharData(const XML_Char*, int) {}
int bodyIdx() const {
for (int i = static_cast<int>(stack.size()) - 1; i >= 0; i--) {
if (stack[i].tag == "body") {
+32 -22
View File
@@ -47,33 +47,43 @@ struct ReverseState : StackState {
const char* bestTierName = nullptr;
ReverseState(const int spineIndex, const std::string& xpath) : spineIndex(spineIndex) {
// Parse optional /text()[N].M suffix before normalizing for element matching.
// Parse optional text-node suffix before normalizing for element matching.
// KOReader emits two shapes that both land here:
// /text()[N].M — explicit 1-based text-node index + codepoint offset
// /text().M — implicit first text-node (N=1) + codepoint offset
std::string raw = xpath;
for (char& c : raw) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
const std::string tnPat = "/text()[";
const std::string tnPat = "/text()";
const size_t tnPos = raw.rfind(tnPat);
if (tnPos != std::string::npos) {
const size_t numStart = tnPos + tnPat.size();
size_t numEnd = numStart;
while (numEnd < raw.size() && std::isdigit(static_cast<unsigned char>(raw[numEnd]))) {
numEnd++;
size_t cursor = tnPos + tnPat.size();
int nodeIdx = 1;
bool valid = true;
if (cursor < raw.size() && raw[cursor] == '[') {
cursor++;
size_t numEnd = cursor;
while (numEnd < raw.size() && std::isdigit(static_cast<unsigned char>(raw[numEnd]))) {
numEnd++;
}
if (numEnd > cursor && numEnd < raw.size() && raw[numEnd] == ']') {
nodeIdx = static_cast<int>(std::strtol(raw.substr(cursor, numEnd - cursor).c_str(), nullptr, 10));
cursor = numEnd + 1;
} else {
valid = false;
}
}
if (numEnd > numStart && numEnd < raw.size() && raw[numEnd] == ']') {
const long nodeIdx = std::strtol(raw.substr(numStart, numEnd - numStart).c_str(), nullptr, 10);
if (nodeIdx >= 1) {
targetTextNodeIndex = static_cast<int>(nodeIdx);
size_t after = numEnd + 1;
if (after < raw.size() && raw[after] == '.') {
after++;
size_t charEnd = after;
while (charEnd < raw.size() && std::isdigit(static_cast<unsigned char>(raw[charEnd]))) {
charEnd++;
}
if (charEnd > after) {
const long charOff = std::strtol(raw.substr(after, charEnd - after).c_str(), nullptr, 10);
if (charOff >= 0) {
targetCharOffset = static_cast<int>(charOff);
}
if (valid && nodeIdx >= 1) {
targetTextNodeIndex = nodeIdx;
if (cursor < raw.size() && raw[cursor] == '.') {
cursor++;
size_t charEnd = cursor;
while (charEnd < raw.size() && std::isdigit(static_cast<unsigned char>(raw[charEnd]))) {
charEnd++;
}
if (charEnd > cursor) {
const long charOff = std::strtol(raw.substr(cursor, charEnd - cursor).c_str(), nullptr, 10);
if (charOff >= 0) {
targetCharOffset = static_cast<int>(charOff);
}
}
}
+1 -1
View File
@@ -519,7 +519,7 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
// kosync convention: no stored progress for the document is signalled by
// HTTP 200 with an empty body ("{}"), not by 404. Detect that here so the
// caller doesn't apply a zeroed-out position as if it were real progress.
if (doc["document"].isNull() || doc["progress"].isNull()) {
if (doc["progress"].isNull()) {
std::string jsonDump;
serializeJson(doc, jsonDump);
LOG_DBG("KOSync", "Empty progress payload — treating as not found | payload=%s", jsonDump.c_str());
+17 -4
View File
@@ -58,10 +58,23 @@ KOReaderPosition ProgressMapper::toKOReader(const std::shared_ptr<Epub>& epub, c
result.percentage = epub->calculateProgress(pos.spineIndex, intraSpineProgress);
// Generate XPath for the current position.
// Always use the indexer which SAX-parses the actual XHTML to find the correct
// element path — a naive "/body/DocFragment[N]/body/p[M]" would assume paragraphs
// are direct children of <body>, which breaks for wrapped chapters (e.g. div/section).
result.xpath = ChapterXPathIndexer::findXPathForProgress(epub, pos.spineIndex, intraSpineProgress);
// When we have a paragraph index from the section LUT, target that specific <p> element
// directly — this produces a structurally precise full-ancestry path even for chapters
// where paragraphs are nested inside divs/sections. Fall back to the progress-based
// scan (which works for any content) when no paragraph index is available.
if (pos.hasParagraphIndex && pos.paragraphIndex > 0) {
// When a seek hint is set, the LUT entry's paragraphIndex equals pos.paragraphIndex
// (both describe the same page). The byte offset now points at the body-child element
// that was current at the page break, so re-parsing from there will re-encounter that
// paragraph — seed startParagraphCount with paragraphIndex-1 to avoid double counting.
const uint16_t startCount =
pos.xhtmlSeekHint > 0 && pos.paragraphIndex > 0 ? static_cast<uint16_t>(pos.paragraphIndex - 1) : 0;
result.xpath = ChapterXPathIndexer::findXPathForParagraph(epub, pos.spineIndex, pos.paragraphIndex,
pos.xhtmlSeekHint, startCount);
}
if (result.xpath.empty()) {
result.xpath = ChapterXPathIndexer::findXPathForProgress(epub, pos.spineIndex, intraSpineProgress);
}
if (result.xpath.empty()) {
result.xpath = generateXPath(pos.spineIndex);
}
+3 -2
View File
@@ -11,8 +11,9 @@ struct CrossPointPosition {
int spineIndex; // Current spine item (chapter) index
int pageNumber; // Current page within the spine item (estimated if no paragraph LUT)
int totalPages; // Total pages in the current spine item
uint16_t paragraphIndex = 0; // 1-based <p> index from XPath (0 if unavailable)
bool hasParagraphIndex = false; // True when paragraphIndex was resolved from XPath
uint16_t paragraphIndex = 0; // 1-based <p> index (0 if unavailable)
bool hasParagraphIndex = false; // True when paragraphIndex is valid
uint32_t xhtmlSeekHint = 0; // Byte offset hint for findXPathForParagraph (0 = no hint)
};
/**
+2
View File
@@ -40,6 +40,7 @@ struct KOReaderSyncSessionState {
int totalPagesInSpine = 0;
uint16_t paragraphIndex = 0;
bool hasParagraphIndex = false;
uint32_t xhtmlSeekHint = 0; // byte offset hint for findXPathForParagraph (0 = no hint)
KOReaderSyncIntentState intent = KOReaderSyncIntentState::COMPARE;
KOReaderSyncOutcomeState outcome = KOReaderSyncOutcomeState::NONE;
int resultSpineIndex = 0;
@@ -55,6 +56,7 @@ struct KOReaderSyncSessionState {
totalPagesInSpine = 0;
paragraphIndex = 0;
hasParagraphIndex = false;
xhtmlSeekHint = 0;
intent = KOReaderSyncIntentState::COMPARE;
outcome = KOReaderSyncOutcomeState::NONE;
resultSpineIndex = 0;
+2
View File
@@ -82,6 +82,7 @@ bool JsonSettingsIO::saveState(const CrossPointState& s, const char* path) {
sync["totalPagesInSpine"] = s.koReaderSyncSession.totalPagesInSpine;
sync["paragraphIndex"] = s.koReaderSyncSession.paragraphIndex;
sync["hasParagraphIndex"] = s.koReaderSyncSession.hasParagraphIndex;
sync["xhtmlSeekHint"] = s.koReaderSyncSession.xhtmlSeekHint;
sync["intent"] = static_cast<uint8_t>(s.koReaderSyncSession.intent);
sync["outcome"] = static_cast<uint8_t>(s.koReaderSyncSession.outcome);
sync["resultSpineIndex"] = s.koReaderSyncSession.resultSpineIndex;
@@ -134,6 +135,7 @@ bool JsonSettingsIO::loadState(CrossPointState& s, const char* json) {
s.koReaderSyncSession.totalPagesInSpine = sync["totalPagesInSpine"] | 0;
s.koReaderSyncSession.paragraphIndex = sync["paragraphIndex"] | (uint16_t)0;
s.koReaderSyncSession.hasParagraphIndex = sync["hasParagraphIndex"] | false;
s.koReaderSyncSession.xhtmlSeekHint = sync["xhtmlSeekHint"] | (uint32_t)0;
s.koReaderSyncSession.intent =
static_cast<KOReaderSyncIntentState>(sync["intent"] | static_cast<uint8_t>(KOReaderSyncIntentState::COMPARE));
s.koReaderSyncSession.outcome =
+1 -1
View File
@@ -285,7 +285,7 @@ void ActivityManager::goToKOReaderSync() {
replaceActivity(std::make_unique<KOReaderSyncActivity>(renderer, mappedInput, sync.epubPath, sync.spineIndex,
sync.page, sync.totalPagesInSpine, sync.paragraphIndex,
sync.hasParagraphIndex, sync.intent));
sync.hasParagraphIndex, sync.xhtmlSeekHint, sync.intent));
}
void ActivityManager::replaceWithReader(std::string path, ReturnHint hint) {
+20 -2
View File
@@ -637,8 +637,26 @@ void EpubReaderActivity::launchKOReaderSync(const SyncLaunchMode mode) {
sync.spineIndex = currentSpineIndex;
sync.page = currentPage;
sync.totalPagesInSpine = totalPages;
sync.paragraphIndex = 0;
sync.hasParagraphIndex = false;
// Populate paragraph index and XHTML seek hint from section LUT if available.
if (section) {
if (const auto pIdx = section->getParagraphIndexForPage(static_cast<uint16_t>(currentPage))) {
sync.paragraphIndex = *pIdx;
sync.hasParagraphIndex = true;
if (const auto hint = section->getXhtmlByteOffsetForPage(static_cast<uint16_t>(currentPage))) {
sync.xhtmlSeekHint = *hint;
} else {
sync.xhtmlSeekHint = 0;
}
} else {
sync.paragraphIndex = 0;
sync.hasParagraphIndex = false;
sync.xhtmlSeekHint = 0;
}
} else {
sync.paragraphIndex = 0;
sync.hasParagraphIndex = false;
sync.xhtmlSeekHint = 0;
}
sync.intent = syncIntent;
sync.outcome = KOReaderSyncOutcomeState::PENDING;
sync.resultSpineIndex = 0;
@@ -651,8 +651,8 @@ bool KOReaderSyncActivity::computeLocalProgressAndChapter() {
return false;
}
CrossPointPosition localPos = {currentSpineIndex, currentPage, totalPagesInSpine, localParagraphIndex,
hasLocalParagraphIndex};
CrossPointPosition localPos = {currentSpineIndex, currentPage, totalPagesInSpine, localParagraphIndex,
hasLocalParagraphIndex, localXhtmlSeekHint};
localProgress = ProgressMapper::toKOReader(epub, localPos);
const int localTocIndex = epub->getTocIndexForSpineIndex(currentSpineIndex);
+3 -1
View File
@@ -34,7 +34,7 @@ class KOReaderSyncActivity final : public Activity {
public:
explicit KOReaderSyncActivity(GfxRenderer& renderer, MappedInputManager& mappedInput, const std::string& epubPath,
int currentSpineIndex, int currentPage, int totalPagesInSpine,
uint16_t paragraphIndex = 0, bool hasParagraphIndex = false,
uint16_t paragraphIndex = 0, bool hasParagraphIndex = false, uint32_t xhtmlSeekHint = 0,
KOReaderSyncIntentState syncIntent = KOReaderSyncIntentState::COMPARE)
: Activity("KOReaderSync", renderer, mappedInput),
epubPath(epubPath),
@@ -43,6 +43,7 @@ class KOReaderSyncActivity final : public Activity {
totalPagesInSpine(totalPagesInSpine),
localParagraphIndex(paragraphIndex),
hasLocalParagraphIndex(hasParagraphIndex),
localXhtmlSeekHint(xhtmlSeekHint),
syncIntent(syncIntent),
remoteProgress{},
remotePosition{},
@@ -75,6 +76,7 @@ class KOReaderSyncActivity final : public Activity {
int totalPagesInSpine;
uint16_t localParagraphIndex;
bool hasLocalParagraphIndex;
uint32_t localXhtmlSeekHint;
KOReaderSyncIntentState syncIntent = KOReaderSyncIntentState::COMPARE;
State state = WIFI_SELECTION;