Merge remote-tracking branch 'origin/develop' into feat-deferred-refresh

lib/KOReaderSync/KOReaderCredentialStore.cpp

@@ -5,16 +5,27 @@
 #include <ObfuscationUtils.h>
 
 namespace {
-// Default sync server URL
-constexpr char DEFAULT_SERVER_URL[] = "https://sync.koreader.rocks:443";
+// Default sync server URL. crosspoint-sync speaks the full KOSync protocol, so
+// pointing at any other kosync server (e.g. https://sync.koreader.rocks:443)
+// still works via the custom server URL setting.
+constexpr char DEFAULT_SERVER_URL[] = "https://sync.crosspointreader.com";
+
+// Default before config version 2. Configs saved without a version stamp and an
+// empty serverUrl were implicitly syncing here — they get pinned on upgrade.
+constexpr char LEGACY_DEFAULT_SERVER_URL[] = "https://sync.koreader.rocks:443";
+
+// Bumped when a change to defaults would alter behavior for existing configs.
+constexpr uint8_t CONFIG_VERSION = 2;
 }  // namespace
 
 void KOReaderCredentialStore::toJson(JsonDocument& doc) const {
+  doc["cfgVersion"] = CONFIG_VERSION;
   doc["username"] = getUsername();
   doc["password_obf"] = obfuscation::obfuscateToBase64(getPassword());
   doc["serverUrl"] = getServerUrl();
   doc["matchMethod"] = static_cast<uint8_t>(getMatchMethod());
   doc["sendMetadata"] = getSendMetadata();
+  doc["syncBehavior"] = static_cast<uint8_t>(getSyncBehavior());
 }
 
 bool KOReaderCredentialStore::fromJson(JsonVariantConst doc) {
@@ -26,6 +37,19 @@ bool KOReaderCredentialStore::fromJson(JsonVariantConst doc) {
   setCredentials(user, pass);
   setServerUrl(doc["serverUrl"] | "");
 
+  // The default server changed in config v2 (sync.koreader.rocks -> crosspoint-sync).
+  // A pre-v2 config with credentials and no explicit URL was actively syncing
+  // against the old default — pin that URL so the upgrade doesn't switch servers
+  // out from under the user. Fresh setups get the new default.
+  const uint8_t cfgVersion = doc["cfgVersion"] | (uint8_t)1;
+  if (cfgVersion < CONFIG_VERSION) {
+    if (getServerUrl().empty() && hasCredentials()) {
+      LOG_DBG("KRS", "Pre-v2 config used the old default server; pinning %s", LEGACY_DEFAULT_SERVER_URL);
+      setServerUrl(LEGACY_DEFAULT_SERVER_URL);
+    }
+    needsResave = true;  // stamp cfgVersion so this migration runs once
+  }
+
   uint8_t method = doc["matchMethod"] | (uint8_t)0;
   if (method <= static_cast<uint8_t>(DocumentMatchMethod::BINARY)) {
     setMatchMethod(static_cast<DocumentMatchMethod>(method));
@@ -35,6 +59,18 @@ bool KOReaderCredentialStore::fromJson(JsonVariantConst doc) {
   }
   setSendMetadata(doc["sendMetadata"] | false);
 
+  const JsonVariantConst behaviorValue = doc["syncBehavior"];
+  const bool missingBehavior = behaviorValue.isNull();
+  uint8_t behavior = behaviorValue | static_cast<uint8_t>(KOReaderSyncBehavior::ASK_EVERY_TIME);
+  if (behavior <= static_cast<uint8_t>(KOReaderSyncBehavior::SMART)) {
+    setSyncBehavior(static_cast<KOReaderSyncBehavior>(behavior));
+    needsResave = needsResave || missingBehavior;
+  } else {
+    LOG_DBG("KRS", "Invalid syncBehavior %u in JSON, resetting to ASK_EVERY_TIME", behavior);
+    setSyncBehavior(KOReaderSyncBehavior::ASK_EVERY_TIME);
+    needsResave = true;
+  }
+
   if (needsResave) {
     LOG_DBG("KRS", "Resaved KOReader credentials to update format");
     saveToFile();
@@ -105,3 +141,11 @@ void KOReaderCredentialStore::setSendMetadata(bool enabled) {
   sendMetadata = enabled;
   LOG_DBG("KRS", "Set send metadata: %s", enabled ? "true" : "false");
 }
+
+void KOReaderCredentialStore::setSyncBehavior(KOReaderSyncBehavior behavior) {
+  if (static_cast<uint8_t>(behavior) > static_cast<uint8_t>(KOReaderSyncBehavior::SMART)) {
+    behavior = KOReaderSyncBehavior::ASK_EVERY_TIME;
+  }
+  syncBehavior = behavior;
+  LOG_DBG("KRS", "Set sync behavior: %s", behavior == KOReaderSyncBehavior::SMART ? "Smart" : "Ask");
+}

lib/KOReaderSync/KOReaderCredentialStore.h

@@ -11,6 +11,12 @@ enum class DocumentMatchMethod : uint8_t {
   BINARY = 1,    // Match by partial MD5 of file content (more accurate, but files must be identical)
 };
 
+// How manual "Sync Progress" resolves differences after fetching remote progress.
+enum class KOReaderSyncBehavior : uint8_t {
+  ASK_EVERY_TIME = 0,  // Preserve legacy behavior: always show Apply/Upload choices.
+  SMART = 1,           // Auto-resolve simple cases using furthest progress.
+};
+
 /**
  * Singleton class for storing KOReader sync credentials on the SD card.
  * Passwords are XOR-obfuscated with the device's unique hardware MAC address
@@ -25,6 +31,7 @@ class KOReaderCredentialStore : public PersistableStore<KOReaderCredentialStore>
   std::string serverUrl;                                            // Custom sync server URL (empty = default)
   DocumentMatchMethod matchMethod = DocumentMatchMethod::FILENAME;  // Default to filename for compatibility
   bool sendMetadata = false;                                        // Send document metadata with progress sync
+  KOReaderSyncBehavior syncBehavior = KOReaderSyncBehavior::SMART;
 
   // Private constructor for singleton
   KOReaderCredentialStore() = default;
@@ -65,6 +72,10 @@ class KOReaderCredentialStore : public PersistableStore<KOReaderCredentialStore>
   // Send metadata setting
   void setSendMetadata(bool enabled);
   bool getSendMetadata() const { return sendMetadata; }
+
+  // Sync behavior
+  void setSyncBehavior(KOReaderSyncBehavior behavior);
+  KOReaderSyncBehavior getSyncBehavior() const { return syncBehavior; }
 };
 
 // Helper macro to access credential store

lib/KOReaderSync/KOReaderSyncClient.cpp

@@ -92,6 +92,43 @@ KOReaderSyncClient::Error KOReaderSyncClient::authenticate() {
   return SERVER_ERROR;
 }
 
+KOReaderSyncClient::Error KOReaderSyncClient::createUser() {
+  lastHttpCode = 0;
+  if (!KOREADER_STORE.hasCredentials()) {
+    LOG_DBG("KOSync", "No credentials configured");
+    return NO_CREDENTIALS;
+  }
+
+  const std::string url = KOREADER_STORE.getBaseUrl() + "/users/create";
+  LOG_DBG("KOSync", "Creating account: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
+  if (insufficientHeap()) return LOW_MEMORY;
+
+  JsonDocument doc;
+  doc["username"] = KOREADER_STORE.getUsername();
+  doc["password"] = KOREADER_STORE.getMd5Password();
+  std::string body;
+  serializeJson(doc, body);
+
+  freeink::SecureHttpClient http;
+  http.setInsecure();
+  if (!http.begin(url)) {
+    LOG_ERR("KOSync", "Bad URL: %s", url.c_str());
+    return NETWORK_ERROR;
+  }
+  http.addHeader("Accept", "application/vnd.koreader.v1+json");
+  http.addHeader("Content-Type", "application/json");
+  const int httpCode = http.sendRequest("POST", body);
+  http.end();
+  lastHttpCode = httpCode;
+
+  LOG_DBG("KOSync", "Create user response: %d", httpCode);
+
+  if (httpCode <= 0) return NETWORK_ERROR;
+  if (httpCode == 200 || httpCode == 201) return OK;
+  if (httpCode == 402) return USER_EXISTS;
+  return SERVER_ERROR;
+}
+
 KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& documentHash,
                                                           KOReaderProgress& outProgress) {
   lastHttpCode = 0;
@@ -138,6 +175,24 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
     outProgress.deviceId = doc["device_id"].as<std::string>();
     outProgress.timestamp = doc["timestamp"].as<int64_t>();
 
+    // Extended crosspoint-sync field; absent on plain kosync servers.
+    outProgress.position.reset();
+    const JsonObjectConst pos = doc["position"].as<JsonObjectConst>();
+    if (!pos.isNull()) {
+      KOReaderRichPosition rich;
+      rich.pctQ = pos["pctQ"].as<uint32_t>();
+      rich.spineIndex = pos["spine"].as<uint16_t>();
+      rich.pageNumber = pos["page"].as<uint16_t>();
+      const uint16_t pages = pos["pages"].as<uint16_t>();
+      rich.totalPages = pages > 0 ? pages : 1;
+      const uint16_t para = pos["para"].as<uint16_t>();
+      if (para > 0) rich.paragraphIndex = para;
+      rich.xpath = pos["xpath"].as<const char*>() ? pos["xpath"].as<const char*>() : "";
+      LOG_DBG("KOSync", "Got rich position: spine=%u page=%u/%u para=%u", rich.spineIndex, rich.pageNumber,
+              rich.totalPages, para);
+      outProgress.position = std::move(rich);
+    }
+
     LOG_DBG("KOSync", "Got progress: %.2f%% at %s", outProgress.percentage * 100, outProgress.progress.c_str());
     return OK;
   }
@@ -172,6 +227,18 @@ KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgr
   doc["percentage"] = progress.percentage;
   doc["device"] = DEVICE_NAME;
   doc["device_id"] = DEVICE_ID;
+  if (progress.position.has_value()) {
+    // Extended crosspoint-sync field; kosync servers ignore unknown keys.
+    const auto& p = *progress.position;
+    auto pos = doc["position"].to<JsonObject>();
+    pos["pctQ"] = p.pctQ;
+    pos["spine"] = p.spineIndex;
+    pos["page"] = p.pageNumber;
+    pos["pages"] = p.totalPages;
+    if (p.paragraphIndex.has_value()) pos["para"] = *p.paragraphIndex;
+    // Server rejects the whole position object if xpath exceeds 120 bytes.
+    if (!p.xpath.empty() && p.xpath.size() <= 120) pos["xpath"] = p.xpath;
+  }
 
   std::string body;
   serializeJson(doc, body);

lib/KOReaderSync/KOReaderSyncClient.h

@@ -14,17 +14,33 @@ struct KOReaderMetadata {
   std::string authors;   // Author(s) from EPUB metadata
 };
 
+/**
+ * Rich CrossPoint position sent alongside progress uploads. Maps 1:1 onto the
+ * crosspoint-sync extended `position` object (see crosspoint-sync docs/API.md).
+ * The official KOSync server ignores unknown fields; crosspoint-sync stores it
+ * so CrossPoint<->CrossPoint sync is lossless instead of xpath-approximated.
+ */
+struct KOReaderRichPosition {
+  uint32_t pctQ = 0;                       // Percentage quantized 0..1,000,000 (authoritative)
+  uint16_t spineIndex = 0;                 // Spine (chapter) index
+  uint16_t pageNumber = 0;                 // Page within spine (layout-dependent hint)
+  uint16_t totalPages = 1;                 // Spine page count (layout-dependent hint)
+  std::optional<uint16_t> paragraphIndex;  // Synthetic 1-based paragraph index
+  std::string xpath;                       // KOReader-style xpath (server cap: 120 bytes)
+};
+
 /**
  * Progress data from KOReader sync server.
  */
 struct KOReaderProgress {
-  std::string document;                      // Document hash
-  std::string progress;                      // XPath-like progress string
-  float percentage;                          // Progress percentage (0.0 to 1.0)
-  std::string device;                        // Device name
-  std::string deviceId;                      // Device ID
-  int64_t timestamp;                         // Unix timestamp of last update
-  std::optional<KOReaderMetadata> metadata;  // Optional document metadata
+  std::string document;                          // Document hash
+  std::string progress;                          // XPath-like progress string
+  float percentage;                              // Progress percentage (0.0 to 1.0)
+  std::string device;                            // Device name
+  std::string deviceId;                          // Device ID
+  int64_t timestamp;                             // Unix timestamp of last update
+  std::optional<KOReaderMetadata> metadata;      // Optional document metadata
+  std::optional<KOReaderRichPosition> position;  // Optional rich position (crosspoint-sync servers only)
 };
 
 /**
@@ -43,7 +59,17 @@ struct KOReaderProgress {
  */
 class KOReaderSyncClient {
  public:
-  enum Error { OK = 0, NO_CREDENTIALS, NETWORK_ERROR, AUTH_FAILED, SERVER_ERROR, JSON_ERROR, NOT_FOUND, LOW_MEMORY };
+  enum Error {
+    OK = 0,
+    NO_CREDENTIALS,
+    NETWORK_ERROR,
+    AUTH_FAILED,
+    SERVER_ERROR,
+    JSON_ERROR,
+    NOT_FOUND,
+    LOW_MEMORY,
+    USER_EXISTS
+  };
 
   /**
    * Authenticate with the sync server (validate credentials).
@@ -51,6 +77,14 @@ class KOReaderSyncClient {
    */
   static Error authenticate();
 
+  /**
+   * Register a new account on the sync server using the stored credentials
+   * (POST /users/create with the MD5 auth key — the server never sees the
+   * plain password).
+   * @return OK on success, USER_EXISTS if the username is taken
+   */
+  static Error createUser();
+
   /**
    * Get reading progress for a document.
    * @param documentHash The document hash (from KOReaderDocumentId)

lib/KOReaderSync/ProgressMapper.cpp

@@ -724,6 +724,59 @@ SavedProgressPosition ProgressMapper::toSavedProgress(const std::shared_ptr<Epub
   return result;
 }
 
+std::optional<CrossPointPosition> ProgressMapper::fromRichPosition(const std::shared_ptr<Epub>& epub,
+                                                                   const KOReaderRichPosition& rich,
+                                                                   GfxRenderer& renderer) {
+  const int spineCount = epub->getSpineItemsCount();
+  if (static_cast<int>(rich.spineIndex) >= spineCount) {
+    LOG_DBG("PM", "Rich position spine %u out of range (%d spine items)", rich.spineIndex, spineCount);
+    return std::nullopt;
+  }
+
+  CrossPointPosition result{};
+  result.spineIndex = rich.spineIndex;
+
+  Section tempSection(epub, result.spineIndex, renderer);
+  const auto cachedCount = tempSection.getCachedPageCount();
+  if (!cachedCount || *cachedCount <= 0) {
+    // No local layout for the target spine yet; the percentage/xpath mapping
+    // handles density estimation better than a blind copy of remote pages.
+    LOG_DBG("PM", "Rich position spine %u has no cached page count", rich.spineIndex);
+    return std::nullopt;
+  }
+  result.totalPages = *cachedCount;
+
+  const int remotePages = rich.totalPages > 0 ? rich.totalPages : 1;
+  if (result.totalPages == remotePages) {
+    // Identical layout (same render settings) — the page transfers losslessly.
+    result.pageNumber = std::min<int>(rich.pageNumber, result.totalPages - 1);
+    LOG_DBG("PM", "Rich position exact: spine=%d page=%d/%d", result.spineIndex, result.pageNumber, result.totalPages);
+    return result;
+  }
+
+  // Layout differs; the paragraph LUT is the most accurate anchor we have.
+  if (rich.paragraphIndex.has_value()) {
+    const auto lutPage = tempSection.getPageForParagraphIndex(*rich.paragraphIndex);
+    if (lutPage.has_value()) {
+      result.paragraphIndex = *rich.paragraphIndex;
+      result.hasParagraphIndex = true;
+      result.pageNumber = std::min<int>(*lutPage, result.totalPages - 1);
+      LOG_DBG("PM", "Rich position para %u -> spine=%d page=%d/%d", *rich.paragraphIndex, result.spineIndex,
+              result.pageNumber, result.totalPages);
+      return result;
+    }
+  }
+
+  // Fall back to the intra-spine page fraction.
+  const float intra =
+      (remotePages > 1) ? static_cast<float>(rich.pageNumber) / static_cast<float>(remotePages - 1) : 0.0f;
+  result.pageNumber = std::max(
+      0, std::min(static_cast<int>(intra * static_cast<float>(result.totalPages - 1) + 0.5f), result.totalPages - 1));
+  LOG_DBG("PM", "Rich position scaled: spine=%d remote %u/%d -> page=%d/%d", result.spineIndex, rich.pageNumber,
+          remotePages, result.pageNumber, result.totalPages);
+  return result;
+}
+
 CrossPointPosition ProgressMapper::toCrossPoint(const std::shared_ptr<Epub>& epub, const SavedProgressPosition& koPos,
                                                 GfxRenderer& renderer, int currentSpineIndex,
                                                 int totalPagesInCurrentSpine, int fallbackTotalPages) {

lib/KOReaderSync/ProgressMapper.h

@@ -3,8 +3,11 @@
 #include <GfxRenderer.h>
 
 #include <memory>
+#include <optional>
 #include <string>
 
+#include "KOReaderSyncClient.h"
+
 /**
  * CrossPoint position representation.
  */
@@ -65,6 +68,20 @@ class ProgressMapper {
                                          GfxRenderer& renderer, int currentSpineIndex = -1,
                                          int totalPagesInCurrentSpine = 0, int fallbackTotalPages = 0);
 
+  /**
+   * Convert a rich CrossPoint position (downloaded from a crosspoint-sync
+   * server) directly to a CrossPoint position, without XPath approximation.
+   * When the local layout matches the uploader's (same spine page count) the
+   * page transfers losslessly; otherwise the paragraph LUT or the intra-spine
+   * page fraction is used.
+   *
+   * @return The position, or std::nullopt when the rich position cannot be
+   *         applied (spine out of range, no section cache) and the caller
+   *         should fall back to toCrossPoint().
+   */
+  static std::optional<CrossPointPosition> fromRichPosition(const std::shared_ptr<Epub>& epub,
+                                                            const KOReaderRichPosition& rich, GfxRenderer& renderer);
+
  private:
   /**
    * Generate a fallback XPath by streaming the spine item's XHTML and resolving