Improving KOReader sync

This commit is contained in:
jpirnay
2026-05-10 15:46:33 +02:00
parent b5ea8d3550
commit fe60b12d86
18 changed files with 252 additions and 54 deletions
+43 -3
View File
@@ -12,7 +12,7 @@
#include "parsers/ChapterHtmlSlimParser.h"
namespace {
constexpr uint8_t SECTION_FILE_VERSION = 25;
constexpr uint8_t SECTION_FILE_VERSION = 26;
constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(int) + // fontId
sizeof(float) + // lineCompression
@@ -29,8 +29,11 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(uint32_t) + // anchor map offset
sizeof(uint32_t); // paragraph LUT offset
// On-disk paragraph LUT entry: u32 xhtmlByteOffset + u16 paragraphIndex.
constexpr uint32_t PARAGRAPH_LUT_ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t);
// On-disk paragraph LUT entry: u32 xhtmlByteOffset + u16 paragraphIndex + u16 listItemIndex.
// listItemIndex is the running <li> count at page-break time; together with
// paragraphIndex it lets KOReader-supplied <p>- and <li>-anchored XPaths snap to
// the exact page on download.
constexpr uint32_t PARAGRAPH_LUT_ENTRY_SIZE = sizeof(uint32_t) + sizeof(uint16_t) + sizeof(uint16_t);
inline uint32_t paragraphLutEntryOffset(uint32_t lutStart, uint16_t page) {
return lutStart + page * PARAGRAPH_LUT_ENTRY_SIZE;
}
@@ -471,6 +474,7 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
for (const auto& entry : paragraphLut) {
serialization::writePod(file, entry.xhtmlByteOffset);
serialization::writePod(file, entry.paragraphIndex);
serialization::writePod(file, entry.listItemIndex);
}
// Patch header with final pageCount, lutOffset, anchorMapOffset, and paragraphLutOffset
@@ -803,6 +807,42 @@ std::optional<uint16_t> Section::getParagraphIndexForPage(const uint16_t page) c
return pIdx;
}
std::optional<uint16_t> Section::getPageForListItemIndex(const uint16_t liIndex) const {
if (liIndex == 0) {
return std::nullopt;
}
FsFile f;
uint16_t count = 0;
uint32_t lutStart = 0;
if (!readParagraphLutHeader(f, count, lutStart)) {
return std::nullopt;
}
const uint32_t fileSize = f.size();
// Mirror getPageForParagraphIndex: each entry stores the running li count at page-break
// time, so the target li first appears on the smallest i where storedLiIdx[i] >= liIndex.
// The listItemIndex field follows xhtmlByteOffset + paragraphIndex within each entry.
for (uint16_t i = 0; i < count; i++) {
const uint32_t entryOffset = paragraphLutEntryOffset(lutStart, i) + sizeof(uint32_t) + sizeof(uint16_t);
const uint64_t requiredOffset = static_cast<uint64_t>(entryOffset) + sizeof(uint16_t);
if (requiredOffset > fileSize) {
f.close();
return std::nullopt;
}
f.seek(entryOffset);
uint16_t pageLiIdx;
serialization::readPod(f, pageLiIdx);
if (pageLiIdx >= liIndex) {
f.close();
return i;
}
}
f.close();
return static_cast<uint16_t>(count - 1);
}
std::optional<uint32_t> Section::getXhtmlByteOffsetForPage(const uint16_t page) const {
FsFile f;
uint16_t count = 0;
+6
View File
@@ -90,6 +90,12 @@ class Section {
// Returns nullopt if the paragraph LUT is not available (old cache format).
std::optional<uint16_t> getPageForParagraphIndex(uint16_t pIndex) const;
// Look up the page number for a running <li> index (1-based, the Nth <li> at any depth
// in the chapter). Used to snap KOReader-supplied list-item XPaths to a precise page
// the same way getPageForParagraphIndex handles <p>-anchored XPaths.
// Returns nullopt if the LUT is not available or the index is out of range.
std::optional<uint16_t> getPageForListItemIndex(uint16_t liIndex) const;
// Look up the paragraph index for a given page number.
// Returns the 1-based paragraph index of the last <p> element on or before the page.
// Returns nullopt if the paragraph LUT is not available (old cache format).
@@ -250,7 +250,7 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() {
// Callers must ensure currentPage is non-null and carries content; the helper resets
// currentPage to a fresh Page and zeroes currentPageNextY so the caller can keep building.
void ChapterHtmlSlimParser::emitPage(uint32_t xhtmlByteOffset) {
paragraphLutPerPage.push_back({xhtmlByteOffset, xpathParagraphIndex});
paragraphLutPerPage.push_back({xhtmlByteOffset, xpathParagraphIndex, xpathListItemIndex});
completePageFn(std::move(currentPage));
completedPageCount++;
currentPage.reset(new Page());
@@ -814,6 +814,13 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
}
}
// <li> can appear nested inside <ul>/<ol> at any depth, so count it globally —
// not at body-child level. The running count must match what the runtime reverse
// mapper sees so getPageForListItemIndex can snap a KOReader li XPath to a page.
if (self->xpathBodyDepth >= 0 && strcmp(name, "li") == 0) {
self->xpathListItemIndex++;
}
if (matches(name, SKIP_TAGS, NUM_SKIP_TAGS)) {
// start skip
self->skipUntilDepth = self->depth;
@@ -108,7 +108,11 @@ class ChapterHtmlSlimParser final : public Print {
// Stored per page in the section cache so that XPath p[N] can be resolved to a page
// without reparsing, and current page can generate an XPath without reparsing.
uint16_t xpathParagraphIndex = 0; // current <p> sibling index (1-based)
int xpathBodyDepth = -1; // depth of the <body> element (-1 = not yet seen)
// Running count of <li> elements opened anywhere in the chapter (1-based, any depth).
// Used by the section LUT so KOReader-supplied list-item XPaths can snap to the exact
// page on download, the same way <p>-anchored XPaths use xpathParagraphIndex.
uint16_t xpathListItemIndex = 0;
int xpathBodyDepth = -1; // depth of the <body> element (-1 = not yet seen)
// Byte offset of the most recent direct-body-child element start (any tag at xpathBodyDepth+1).
// Recorded at the same depth condition that increments xpathParagraphIndex, so the stored
// offset is guaranteed to land on a body-child element boundary. This keeps the XPath forward
@@ -119,6 +123,7 @@ class ChapterHtmlSlimParser final : public Print {
struct ParagraphLutEntry {
uint32_t xhtmlByteOffset; // byte offset of most recent body-child element start at page break
uint16_t paragraphIndex; // 1-based <p> index at page completion
uint16_t listItemIndex; // running <li> count at page completion (any depth)
};
std::vector<ParagraphLutEntry> paragraphLutPerPage; // deep LUT: one entry per page
+42 -25
View File
@@ -19,7 +19,14 @@ namespace {
// Strategy:
// 1) Count total visible text bytes in chapter.
// 2) Stream parse again and stop when target byte offset is reached.
// 3) Emit either an element path or /text()[N].M when at body text-node level.
// 3) Emit /text()[N].M relative to the deepest open element so KOReader can
// place the cursor at character precision regardless of nesting depth.
//
// Text-node counting matches KOReader/crengine: the Nth XML text node within
// an element, including whitespace-only nodes (those are still real DOM text
// nodes). Empty (len=0) text isn't emitted by expat at all, which mirrors
// KOReader's behavior of skipping the empty text nodes that bare
// <a id="anchor"/> elements would otherwise produce.
struct ForwardState : StackState {
int spineIndex;
@@ -28,20 +35,32 @@ struct ForwardState : StackState {
bool found = false;
XML_Parser parser = nullptr;
int bodyTextNodeCount = 0;
size_t codepointsInBodyTextNode = 0;
bool inBodyTextNode = false;
// Per-element text-node bookkeeping. Mirrors `stack` 1:1 — every push/pop
// appends/removes a counter so the top of the stack always refers to the
// currently open element. `pendingTextNode` is set after every element
// boundary so the next char data starts a fresh text node within whatever
// element is currently on top.
std::vector<int> textNodeIndexStack;
std::vector<size_t> codepointsInTextNodeStack;
bool pendingTextNode = true;
ForwardState(const int spineIndex, const size_t targetOffset) : spineIndex(spineIndex), targetOffset(targetOffset) {}
ForwardState(const int spineIndex, const size_t targetOffset) : spineIndex(spineIndex), targetOffset(targetOffset) {
textNodeIndexStack.reserve(32);
codepointsInTextNodeStack.reserve(32);
}
void onStartElement(const XML_Char* rawName) {
inBodyTextNode = false;
pushElement(rawName);
textNodeIndexStack.push_back(0);
codepointsInTextNodeStack.push_back(0);
pendingTextNode = true;
}
void onEndElement() {
inBodyTextNode = false;
popElement();
if (!textNodeIndexStack.empty()) textNodeIndexStack.pop_back();
if (!codepointsInTextNodeStack.empty()) codepointsInTextNodeStack.pop_back();
pendingTextNode = true;
}
void onCharData(const XML_Char* text, const int len) {
@@ -49,29 +68,29 @@ struct ForwardState : StackState {
return;
}
const bool atBodyLevel = bodyIdx() + 1 == static_cast<int>(stack.size());
if (atBodyLevel && !inBodyTextNode) {
inBodyTextNode = true;
bodyTextNodeCount++;
codepointsInBodyTextNode = 0;
if (pendingTextNode) {
if (!textNodeIndexStack.empty()) textNodeIndexStack.back()++;
if (!codepointsInTextNodeStack.empty()) codepointsInTextNodeStack.back() = 0;
pendingTextNode = false;
}
const size_t cpCount = countUtf8Codepoints(text, len);
if (isWhitespaceOnly(text, len)) {
if (atBodyLevel) {
codepointsInBodyTextNode += countUtf8Codepoints(text, len);
}
if (!codepointsInTextNodeStack.empty()) codepointsInTextNodeStack.back() += cpCount;
return;
}
const size_t visible = countVisibleBytes(text, len);
if (totalTextBytes + visible >= targetOffset) {
if (atBodyLevel && bodyTextNodeCount > 0) {
// KOReader/crengine text-point semantics use codepoint offsets.
const size_t targetVisibleByteInChunk = targetOffset - totalTextBytes;
const size_t cpInChunk = codepointAtVisibleByte(text, len, targetVisibleByteInChunk);
const size_t charOff = codepointsInBodyTextNode + cpInChunk;
result =
currentXPath(spineIndex) + "/text()[" + std::to_string(bodyTextNodeCount) + "]." + std::to_string(charOff);
const int textNode = textNodeIndexStack.empty() ? 0 : textNodeIndexStack.back();
const size_t cpsInNode = codepointsInTextNodeStack.empty() ? 0 : codepointsInTextNodeStack.back();
// KOReader/crengine text-point semantics use codepoint offsets.
const size_t targetVisibleByteInChunk = targetOffset - totalTextBytes;
const size_t cpInChunk = codepointAtVisibleByte(text, len, targetVisibleByteInChunk);
const size_t charOff = cpsInNode + cpInChunk;
if (textNode > 0) {
result = currentXPath(spineIndex) + "/text()[" + std::to_string(textNode) + "]." + std::to_string(charOff);
} else {
result = currentXPath(spineIndex);
}
@@ -83,9 +102,7 @@ struct ForwardState : StackState {
}
totalTextBytes += visible;
if (atBodyLevel) {
codepointsInBodyTextNode += countUtf8Codepoints(text, len);
}
if (!codepointsInTextNodeStack.empty()) codepointsInTextNodeStack.back() += cpCount;
}
};
+2 -2
View File
@@ -29,8 +29,8 @@ std::string ChapterXPathIndexer::findXPathForParagraph(const std::shared_ptr<Epu
bool ChapterXPathIndexer::findProgressForXPath(const std::shared_ptr<Epub>& epub, const int spineIndex,
const std::string& xpath, float& outIntraSpineProgress,
bool& outExactMatch) {
return findProgressForXPathInternal(epub, spineIndex, xpath, outIntraSpineProgress, outExactMatch);
bool& outExactMatch, uint16_t* outListItemIndex) {
return findProgressForXPathInternal(epub, spineIndex, xpath, outIntraSpineProgress, outExactMatch, outListItemIndex);
}
bool ChapterXPathIndexer::tryExtractSpineIndexFromXPath(const std::string& xpath, int& outSpineIndex) {
+6 -1
View File
@@ -46,10 +46,15 @@ class ChapterXPathIndexer {
* @param xpath Incoming KOReader XPath
* @param outIntraSpineProgress Resolved position within spine [0.0, 1.0]
* @param outExactMatch True only for full exact path match
* @param outListItemIndex Optional. When non-null and the target XPath's deepest element
* is /li[N], receives the running <li> count at the matched element so the
* caller can snap to the precise page via Section::getPageForListItemIndex().
* Set to 0 if the target wasn't <li>-anchored.
* @return true if any match was resolved; false means caller should fallback
*/
static bool findProgressForXPath(const std::shared_ptr<Epub>& epub, int spineIndex, const std::string& xpath,
float& outIntraSpineProgress, bool& outExactMatch);
float& outIntraSpineProgress, bool& outExactMatch,
uint16_t* outListItemIndex = nullptr);
/**
* Parse DocFragment index from KOReader-style path segment:
+47 -5
View File
@@ -40,11 +40,22 @@ struct ReverseState : StackState {
size_t codepointsInCurrentTextNode = 0;
int currentTextNodeCount = 0;
// Running <li> count at any depth. Mirrors xpathListItemIndex in ChapterHtmlSlimParser
// so the index captured at match time can be used as a key into the section's li LUT.
int liCount = 0;
// True when the deepest element of targetNorm is /li[N]. Only then is bestLiIndex
// meaningful — for <p>-anchored targets we use the existing paragraph index path.
bool targetEndsInLi = false;
MatchTier bestTier = MatchTier::NONE;
int bestDepth = -1;
size_t bestOffset = 0;
bool bestExact = false;
const char* bestTierName = nullptr;
// Snapshot of liCount at the moment the best match was captured. The reverse
// mapper surfaces this so the runtime can call Section::getPageForListItemIndex()
// and snap a list-item XPath to the precise page on download.
int bestLiIndex = 0;
ReverseState(const int spineIndex, const std::string& xpath) : spineIndex(spineIndex) {
// Parse optional text-node suffix before normalizing for element matching.
@@ -91,11 +102,27 @@ struct ReverseState : StackState {
}
targetNorm = normalizeXPath(xpath);
targetNoIndex = removeIndices(targetNorm);
// Detect /li[N] as the deepest segment of targetNorm. normalizeXPath has already
// stripped any /text() and /text()[N].M suffix and lower-cased the tag names, so
// a simple tail check is enough.
const size_t lastSlash = targetNorm.rfind('/');
if (lastSlash != std::string::npos) {
const std::string tail = targetNorm.substr(lastSlash + 1);
if (tail.size() >= 2 && tail.compare(0, 2, "li") == 0 && (tail.size() == 2 || tail[2] == '[')) {
targetEndsInLi = true;
}
}
}
void onStartElement(const XML_Char* rawName) {
inParentTextNode = false;
pushElement(rawName);
// Increment after pushElement so stack.back().tag is already lowercased and
// matches the parser-side counter, which also fires on startElement.
if (!stack.empty() && stack.back().tag == "li") {
liCount++;
}
}
void onEndElement() {
@@ -134,6 +161,7 @@ struct ReverseState : StackState {
bestOffset = pos;
bestExact = true;
bestTierName = "text-node-exact";
bestLiIndex = liCount;
}
}
codepointsInCurrentTextNode += codepoints;
@@ -191,6 +219,7 @@ struct ReverseState : StackState {
bestOffset = totalTextBytes;
bestExact = isExact;
bestTierName = tierName;
bestLiIndex = liCount;
}
}
};
@@ -198,9 +227,12 @@ struct ReverseState : StackState {
} // namespace
bool findProgressForXPathInternal(const std::shared_ptr<Epub>& epub, const int spineIndex, const std::string& xpath,
float& outIntraSpineProgress, bool& outExactMatch) {
float& outIntraSpineProgress, bool& outExactMatch, uint16_t* outListItemIndex) {
outIntraSpineProgress = 0.0f;
outExactMatch = false;
if (outListItemIndex) {
*outListItemIndex = 0;
}
if (xpath.empty()) {
return false;
@@ -244,13 +276,23 @@ bool findProgressForXPathInternal(const std::shared_ptr<Epub>& epub, const int s
outIntraSpineProgress = std::max(0.0f, std::min(1.0f, outIntraSpineProgress));
}
// Only surface the li index when the target was actually <li>-anchored AND we
// captured a count > 0. NO_IDX fallback tiers can match the wrong <li> sibling,
// so restrict to tiers that imply we were inside the target element itself.
if (outListItemIndex && state.targetEndsInLi && state.bestLiIndex > 0 &&
(state.bestTier == MatchTier::EXACT || state.bestTier == MatchTier::ANCESTOR) &&
state.bestLiIndex <= UINT16_MAX) {
*outListItemIndex = static_cast<uint16_t>(state.bestLiIndex);
}
if (state.targetTextNodeIndex > 0) {
LOG_DBG("KOX", "Reverse: spine=%d %s match textNode=%d char=%d offset=%zu/%zu -> progress=%.3f for '%s'",
LOG_DBG("KOX", "Reverse: spine=%d %s match textNode=%d char=%d offset=%zu/%zu -> progress=%.3f li=%d for '%s'",
spineIndex, state.bestTierName, state.targetTextNodeIndex, state.targetCharOffset, state.bestOffset,
state.totalTextBytes, outIntraSpineProgress, xpath.c_str());
state.totalTextBytes, outIntraSpineProgress, state.bestLiIndex, xpath.c_str());
} else {
LOG_DBG("KOX", "Reverse: spine=%d %s match offset=%zu/%zu -> progress=%.3f for '%s'", spineIndex,
state.bestTierName, state.bestOffset, state.totalTextBytes, outIntraSpineProgress, xpath.c_str());
LOG_DBG("KOX", "Reverse: spine=%d %s match offset=%zu/%zu -> progress=%.3f li=%d for '%s'", spineIndex,
state.bestTierName, state.bestOffset, state.totalTextBytes, outIntraSpineProgress, state.bestLiIndex,
xpath.c_str());
}
return true;
}
+5 -1
View File
@@ -7,7 +7,11 @@
namespace ChapterXPathIndexerInternal {
// outListItemIndex (when non-null) receives the running <li> count at the matched
// element's position whenever the target XPath's deepest element is /li[N]. Set to
// 0 if the target wasn't <li>-anchored or no match was found.
bool findProgressForXPathInternal(const std::shared_ptr<Epub>& epub, int spineIndex, const std::string& xpath,
float& outIntraSpineProgress, bool& outExactMatch);
float& outIntraSpineProgress, bool& outExactMatch,
uint16_t* outListItemIndex = nullptr);
} // namespace ChapterXPathIndexerInternal
+28 -8
View File
@@ -48,10 +48,17 @@ bool resolveFromPercentage(const std::shared_ptr<Epub>& epub, const float percen
KOReaderPosition ProgressMapper::toKOReader(const std::shared_ptr<Epub>& epub, const CrossPointPosition& pos) {
KOReaderPosition result;
// Calculate page progress within current spine item
// Calculate page progress within current spine item.
// Page numbers are 0-based and totalPages is 1-based, so the last page is
// (totalPages - 1). Dividing by (totalPages - 1) maps page 0 to intra=0 and the
// last page to intra=1, which is what KOReader expects when round-tripping.
// Dividing by totalPages would peg the last page short of 100%, so a user who
// finished the chapter would only show ~94% on KOReader.
float intraSpineProgress = 0.0f;
if (pos.totalPages > 0) {
intraSpineProgress = static_cast<float>(pos.pageNumber) / static_cast<float>(pos.totalPages);
if (pos.totalPages > 1) {
intraSpineProgress = static_cast<float>(pos.pageNumber) / static_cast<float>(pos.totalPages - 1);
} else if (pos.totalPages == 1) {
intraSpineProgress = 0.0f;
}
// Calculate overall book progress (0.0-1.0)
@@ -98,11 +105,16 @@ CrossPointPosition ProgressMapper::toCrossPoint(const std::shared_ptr<Epub>& epu
if (ChapterXPathIndexer::tryExtractSpineIndexFromXPath(koPos.xpath, xpathSpineIndex) && xpathSpineIndex >= 0 &&
xpathSpineIndex < spineCount) {
float intraFromXPath = 0.0f;
if (ChapterXPathIndexer::findProgressForXPath(epub, xpathSpineIndex, koPos.xpath, intraFromXPath,
xpathExactMatch)) {
uint16_t liIndexFromXPath = 0;
if (ChapterXPathIndexer::findProgressForXPath(epub, xpathSpineIndex, koPos.xpath, intraFromXPath, xpathExactMatch,
&liIndexFromXPath)) {
result.spineIndex = xpathSpineIndex;
resolvedIntraSpineProgress = intraFromXPath;
usedXPathMapping = true;
if (liIndexFromXPath > 0) {
result.listItemIndex = liIndexFromXPath;
result.hasListItemIndex = true;
}
// KOReader's text-node indexing can differ across renderers/parsers in some
// XHTML shapes. When an XPath-resolved position disagrees materially with
@@ -177,15 +189,23 @@ CrossPointPosition ProgressMapper::toCrossPoint(const std::shared_ptr<Epub>& epu
if (estimatedTotalPages > 0 && resolvedIntraSpineProgress >= 0.0f) {
const float clampedProgress = std::max(0.0f, std::min(1.0f, resolvedIntraSpineProgress));
result.pageNumber = static_cast<int>(clampedProgress * static_cast<float>(estimatedTotalPages));
// Symmetric inverse of the toKOReader formula: intra=1.0 should land on the
// last page (totalPages - 1), intra=0 on page 0. Round-to-nearest avoids
// truncating mid-page progress down to the previous page.
if (estimatedTotalPages > 1) {
result.pageNumber = static_cast<int>(clampedProgress * static_cast<float>(estimatedTotalPages - 1) + 0.5f);
} else {
result.pageNumber = 0;
}
result.pageNumber = std::max(0, std::min(result.pageNumber, estimatedTotalPages - 1));
} else if (spineSize > 0 && estimatedTotalPages > 0) {
result.pageNumber = 0;
}
}
LOG_DBG("ProgressMapper", "Resolved KOReader position: spine=%d intra=%.3f hasPIdx=%s pIdx=%u", result.spineIndex,
resolvedIntraSpineProgress, result.hasParagraphIndex ? "yes" : "no", result.paragraphIndex);
LOG_DBG("ProgressMapper", "Resolved KOReader position: spine=%d intra=%.3f hasPIdx=%s pIdx=%u hasLiIdx=%s liIdx=%u",
result.spineIndex, resolvedIntraSpineProgress, result.hasParagraphIndex ? "yes" : "no", result.paragraphIndex,
result.hasListItemIndex ? "yes" : "no", result.listItemIndex);
const char* mappingSource =
usedXPathMapping ? (usedPercentageReconcile ? "xpath+percentage" : "xpath") : "percentage";
+2
View File
@@ -13,6 +13,8 @@ struct CrossPointPosition {
int totalPages; // Total pages in the current spine item
uint16_t paragraphIndex = 0; // 1-based <p> index (0 if unavailable)
bool hasParagraphIndex = false; // True when paragraphIndex is valid
uint16_t listItemIndex = 0; // 1-based running <li> count when target XPath ends in /li[N]
bool hasListItemIndex = false; // True when listItemIndex is valid
uint32_t xhtmlSeekHint = 0; // Byte offset hint for findXPathForParagraph (0 = no hint)
};