fix: use esp_http_client for KOSync to prevent TLS OOM on ESP32-C3

The Arduino WiFiClientSecure allocates 16KB TLS buffers by default,
which exhausts the ESP32-C3's limited heap (~46KB free after WiFi)
during the TLS handshake. This causes KOReader sync to fail silently
or crash on HTTPS servers (including the default sync.koreader.rocks).

Replace WiFiClientSecure/HTTPClient with esp_http_client (ESP-IDF),
which supports configurable buffer sizes. Use 2KB TLS buffers — more
than sufficient for KOSync's tiny JSON payloads (<1KB).

Also:
- Use esp_crt_bundle for proper TLS certificate verification instead
  of setInsecure()
- Strip trailing slashes from server URL to prevent double-slash in
  API paths
- Add lastHttpCode for diagnostics
- Add heap logging to help debug memory issues

Fixes #581

(cherry picked from commit 835abc19ff14fcbea8571ccb0dfd8d21e882b84c)

lib/KOReaderSync/KOReaderCredentialStore.cpp

@@ -153,16 +153,22 @@ void KOReaderCredentialStore::setServerUrl(const std::string& url) {
 }
 
 std::string KOReaderCredentialStore::getBaseUrl() const {
+  std::string url;
   if (serverUrl.empty()) {
-    return DEFAULT_SERVER_URL;
+    url = DEFAULT_SERVER_URL;
+  } else if (serverUrl.find("://") == std::string::npos) {
+    // Normalize URL: add http:// if no protocol specified (local servers typically don't have SSL)
+    url = "http://" + serverUrl;
+  } else {
+    url = serverUrl;
   }
 
-  // Normalize URL: add http:// if no protocol specified (local servers typically don't have SSL)
-  if (serverUrl.find("://") == std::string::npos) {
-    return "http://" + serverUrl;
+  // Strip trailing slashes to avoid double-slash in API paths
+  while (!url.empty() && url.back() == '/') {
+    url.pop_back();
   }
 
-  return serverUrl;
+  return url;
 }
 
 void KOReaderCredentialStore::setMatchMethod(DocumentMatchMethod method) {

lib/KOReaderSync/KOReaderSyncClient.cpp

@@ -1,31 +1,104 @@
 #include "KOReaderSyncClient.h"
 
 #include <ArduinoJson.h>
-#include <HTTPClient.h>
 #include <Logging.h>
-#include <WiFi.h>
-#include <WiFiClientSecure.h>
+#include <esp_crt_bundle.h>
+#include <esp_http_client.h>
 
 #include <ctime>
 
 #include "KOReaderCredentialStore.h"
 
+int KOReaderSyncClient::lastHttpCode = 0;
+
 namespace {
 // Device identifier for CrossPoint reader
 constexpr char DEVICE_NAME[] = "CrossPoint";
 constexpr char DEVICE_ID[] = "crosspoint-reader";
 
-void addAuthHeaders(HTTPClient& http) {
-  http.addHeader("Accept", "application/vnd.koreader.v1+json");
-  http.addHeader("x-auth-user", KOREADER_STORE.getUsername().c_str());
-  http.addHeader("x-auth-key", KOREADER_STORE.getMd5Password().c_str());
+// Small TLS buffers to fit in ESP32-C3's limited heap (~46KB free after WiFi).
+// KOSync payloads are tiny JSON (<1KB), so 2KB buffers are sufficient.
+// Default 16KB buffers cause OOM during TLS handshake.
+constexpr int HTTP_BUF_SIZE = 2048;
 
-  // HTTP Basic Auth (RFC 7617) header. This is needed to support koreader sync server embedded in Calibre Web Automated
-  // (https://github.com/crocodilestick/Calibre-Web-Automated/blob/main/cps/progress_syncing/protocols/kosync.py)
-  http.setAuthorization(KOREADER_STORE.getUsername().c_str(), KOREADER_STORE.getPassword().c_str());
+// Response buffer for reading HTTP body
+struct ResponseBuffer {
+  char* data = nullptr;
+  int len = 0;
+  int capacity = 0;
+
+  ~ResponseBuffer() { free(data); }
+
+  bool ensure(int size) {
+    if (size <= capacity) return true;
+    char* newData = (char*)realloc(data, size);
+    if (!newData) return false;
+    data = newData;
+    capacity = size;
+    return true;
+  }
+};
+
+// HTTP event handler to collect response body
+esp_err_t httpEventHandler(esp_http_client_event_t* evt) {
+  auto* buf = static_cast<ResponseBuffer*>(evt->user_data);
+  if (evt->event_id == HTTP_EVENT_ON_DATA && buf) {
+    if (buf->ensure(buf->len + evt->data_len + 1)) {
+      memcpy(buf->data + buf->len, evt->data, evt->data_len);
+      buf->len += evt->data_len;
+      buf->data[buf->len] = '\0';
+    }
+  }
+  return ESP_OK;
 }
 
-bool isHttpsUrl(const std::string& url) { return url.rfind("https://", 0) == 0; }
+// Base64 encode for HTTP Basic Auth
+std::string base64Encode(const std::string& input) {
+  static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+  std::string out;
+  out.reserve(((input.size() + 2) / 3) * 4);
+  int val = 0, valb = -6;
+  for (unsigned char c : input) {
+    val = (val << 8) + c;
+    valb += 8;
+    while (valb >= 0) {
+      out.push_back(table[(val >> valb) & 0x3F]);
+      valb -= 6;
+    }
+  }
+  if (valb > -6) out.push_back(table[((val << 8) >> (valb + 8)) & 0x3F]);
+  while (out.size() % 4) out.push_back('=');
+  return out;
+}
+
+// Create configured esp_http_client with small TLS buffers
+esp_http_client_handle_t createClient(const char* url, ResponseBuffer* buf,
+                                       esp_http_client_method_t method = HTTP_METHOD_GET) {
+  esp_http_client_config_t config = {};
+  config.url = url;
+  config.event_handler = httpEventHandler;
+  config.user_data = buf;
+  config.method = method;
+  config.timeout_ms = 15000;
+  config.buffer_size = HTTP_BUF_SIZE;
+  config.buffer_size_tx = HTTP_BUF_SIZE;
+  config.crt_bundle_attach = esp_crt_bundle_attach;
+
+  esp_http_client_handle_t client = esp_http_client_init(&config);
+  if (!client) return nullptr;
+
+  // KOSync auth headers
+  esp_http_client_set_header(client, "Accept", "application/vnd.koreader.v1+json");
+  esp_http_client_set_header(client, "x-auth-user", KOREADER_STORE.getUsername().c_str());
+  esp_http_client_set_header(client, "x-auth-key", KOREADER_STORE.getMd5Password().c_str());
+
+  // HTTP Basic Auth for Calibre-Web-Automated compatibility
+  std::string credentials = KOREADER_STORE.getUsername() + ":" + KOREADER_STORE.getPassword();
+  std::string authHeader = "Basic " + base64Encode(credentials);
+  esp_http_client_set_header(client, "Authorization", authHeader.c_str());
+
+  return client;
+}
 }  // namespace
 
 KOReaderSyncClient::Error KOReaderSyncClient::registerUser() {
@@ -96,33 +169,22 @@ KOReaderSyncClient::Error KOReaderSyncClient::authenticate() {
   }
 
   std::string url = KOREADER_STORE.getBaseUrl() + "/users/auth";
-  LOG_DBG("KOSync", "Authenticating: %s", url.c_str());
+  LOG_DBG("KOSync", "Authenticating: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
 
-  HTTPClient http;
-  std::unique_ptr<WiFiClientSecure> secureClient;
-  WiFiClient plainClient;
+  ResponseBuffer buf;
+  esp_http_client_handle_t client = createClient(url.c_str(), &buf);
+  if (!client) return NETWORK_ERROR;
 
-  if (isHttpsUrl(url)) {
-    secureClient.reset(new WiFiClientSecure);
-    secureClient->setInsecure();
-    http.begin(*secureClient, url.c_str());
-  } else {
-    http.begin(plainClient, url.c_str());
-  }
-  addAuthHeaders(http);
+  esp_err_t err = esp_http_client_perform(client);
+  const int httpCode = esp_http_client_get_status_code(client);
+  lastHttpCode = httpCode;
+  esp_http_client_cleanup(client);
 
-  const int httpCode = http.GET();
-  http.end();
+  LOG_DBG("KOSync", "Auth response: %d (err: %d)", httpCode, err);
 
-  LOG_DBG("KOSync", "Auth response: %d", httpCode);
-
-  if (httpCode == 200) {
-    return OK;
-  } else if (httpCode == 401) {
-    return AUTH_FAILED;
-  } else if (httpCode < 0) {
-    return NETWORK_ERROR;
-  }
+  if (err != ESP_OK) return NETWORK_ERROR;
+  if (httpCode == 200) return OK;
+  if (httpCode == 401) return AUTH_FAILED;
   return SERVER_ERROR;
 }
 
@@ -134,30 +196,24 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
   }
 
   std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress/" + documentHash;
-  LOG_DBG("KOSync", "Getting progress: %s", url.c_str());
+  LOG_DBG("KOSync", "Getting progress: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
 
-  HTTPClient http;
-  std::unique_ptr<WiFiClientSecure> secureClient;
-  WiFiClient plainClient;
+  ResponseBuffer buf;
+  esp_http_client_handle_t client = createClient(url.c_str(), &buf);
+  if (!client) return NETWORK_ERROR;
 
-  if (isHttpsUrl(url)) {
-    secureClient.reset(new WiFiClientSecure);
-    secureClient->setInsecure();
-    http.begin(*secureClient, url.c_str());
-  } else {
-    http.begin(plainClient, url.c_str());
-  }
-  addAuthHeaders(http);
+  esp_err_t err = esp_http_client_perform(client);
+  const int httpCode = esp_http_client_get_status_code(client);
+  lastHttpCode = httpCode;
+  esp_http_client_cleanup(client);
 
-  const int httpCode = http.GET();
+  LOG_DBG("KOSync", "Get progress response: %d (err: %d)", httpCode, err);
 
-  if (httpCode == 200) {
-    // Parse JSON response from response string
-    String responseBody = http.getString();
-    http.end();
+  if (err != ESP_OK) return NETWORK_ERROR;
 
+  if (httpCode == 200 && buf.data) {
     JsonDocument doc;
-    const DeserializationError error = deserializeJson(doc, responseBody);
+    const DeserializationError error = deserializeJson(doc, buf.data);
 
     if (error) {
       LOG_ERR("KOSync", "JSON parse failed: %s", error.c_str());
@@ -175,17 +231,8 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
     return OK;
   }
 
-  http.end();
-
-  LOG_DBG("KOSync", "Get progress response: %d", httpCode);
-
-  if (httpCode == 401) {
-    return AUTH_FAILED;
-  } else if (httpCode == 404) {
-    return NOT_FOUND;
-  } else if (httpCode < 0) {
-    return NETWORK_ERROR;
-  }
+  if (httpCode == 401) return AUTH_FAILED;
+  if (httpCode == 404) return NOT_FOUND;
   return SERVER_ERROR;
 }
 
@@ -196,23 +243,9 @@ KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgr
   }
 
   std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress";
-  LOG_DBG("KOSync", "Updating progress: %s", url.c_str());
+  LOG_DBG("KOSync", "Updating progress: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
 
-  HTTPClient http;
-  std::unique_ptr<WiFiClientSecure> secureClient;
-  WiFiClient plainClient;
-
-  if (isHttpsUrl(url)) {
-    secureClient.reset(new WiFiClientSecure);
-    secureClient->setInsecure();
-    http.begin(*secureClient, url.c_str());
-  } else {
-    http.begin(plainClient, url.c_str());
-  }
-  addAuthHeaders(http);
-  http.addHeader("Content-Type", "application/json");
-
-  // Build JSON body (timestamp not required per API spec)
+  // Build JSON body
   JsonDocument doc;
   doc["document"] = progress.document;
   doc["progress"] = progress.progress;
@@ -225,18 +258,23 @@ KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgr
 
   LOG_DBG("KOSync", "Request body: %s", body.c_str());
 
-  const int httpCode = http.PUT(body.c_str());
-  http.end();
+  ResponseBuffer buf;
+  esp_http_client_handle_t client = createClient(url.c_str(), &buf, HTTP_METHOD_PUT);
+  if (!client) return NETWORK_ERROR;
 
-  LOG_DBG("KOSync", "Update progress response: %d", httpCode);
+  esp_http_client_set_header(client, "Content-Type", "application/json");
+  esp_http_client_set_post_field(client, body.c_str(), body.length());
 
-  if (httpCode == 200 || httpCode == 202) {
-    return OK;
-  } else if (httpCode == 401) {
-    return AUTH_FAILED;
-  } else if (httpCode < 0) {
-    return NETWORK_ERROR;
-  }
+  esp_err_t err = esp_http_client_perform(client);
+  const int httpCode = esp_http_client_get_status_code(client);
+  lastHttpCode = httpCode;
+  esp_http_client_cleanup(client);
+
+  LOG_DBG("KOSync", "Update progress response: %d (err: %d)", httpCode, err);
+
+  if (err != ESP_OK) return NETWORK_ERROR;
+  if (httpCode == 200 || httpCode == 202) return OK;
+  if (httpCode == 401) return AUTH_FAILED;
   return SERVER_ERROR;
 }
 

lib/KOReaderSync/KOReaderSyncClient.h

@@ -74,4 +74,7 @@ class KOReaderSyncClient {
    * Get human-readable error message.
    */
   static const char* errorString(Error error);
+
+  /** HTTP status code from the last request (for diagnostics). */
+  static int lastHttpCode;
 };