fix: use esp_http_client for KOSync to prevent TLS OOM on ESP32-C3 (#1381)

## Summary

* **What is the goal of this PR?** Fix KOReader sync failing on HTTPS
servers due to TLS out-of-memory on ESP32-C3
* **What changes are included?**
- Replace `WiFiClientSecure`/`HTTPClient` with `esp_http_client`
(ESP-IDF native API) for all KOSync HTTP requests
- Use 2KB TLS buffers instead of the default 16KB — KOSync payloads are
tiny JSON (<1KB), so this is more than sufficient
- Use `esp_crt_bundle_attach` for proper TLS certificate verification
(replaces `setInsecure()`)
- Strip trailing slashes from server URL to prevent double-slash in API
paths (e.g. `https://server.com//users/auth`)
  - Add `lastHttpCode` static field for diagnostics
  - Add free heap logging to help debug memory issues

### Problem

The ESP32-C3 has ~46KB free heap after WiFi is initialized.
`WiFiClientSecure` allocates 16KB for TLS RX + 16KB for TLS TX = 32KB
just for the TLS buffers, leaving almost no room for the actual TLS
handshake (which needs additional dynamic allocations). This causes
KOReader sync to:

1. Fail silently with network errors on HTTPS servers (including the
default `sync.koreader.rocks`)
2. Occasionally crash with heap exhaustion

### Solution

`esp_http_client` (the ESP-IDF native HTTP client) allows configuring
`buffer_size` and `buffer_size_tx` independently. Setting both to 2KB
(total 4KB) leaves plenty of heap for the TLS handshake while still
being sufficient for KOSync's small JSON payloads.

This also fixes the `setInsecure()` anti-pattern —
`esp_crt_bundle_attach` provides proper certificate verification using
the ESP-IDF's built-in CA bundle, so credentials and reading history are
no longer sent over unverified TLS connections.

### Files changed

| File | Change |
|------|--------|
| `lib/KOReaderSync/KOReaderSyncClient.cpp` | Replace
WiFiClientSecure/HTTPClient with esp_http_client; add ResponseBuffer,
base64 encoder, createClient helper |
| `lib/KOReaderSync/KOReaderSyncClient.h` | Add `lastHttpCode` static
field |
| `lib/KOReaderSync/KOReaderCredentialStore.cpp` | Strip trailing
slashes from base URL |

## Additional Context

Tested on a CrossPoint X4 (ESP32-C3 with 4MB flash). Before this change,
KOSync auth to `sync.koreader.rocks` (HTTPS) would fail ~80% of the
time. After: works reliably.

The `base64Encode` helper is needed because `esp_http_client` doesn't
have a built-in `setAuthorization()` method like `HTTPClient` does. This
is used for the HTTP Basic Auth header required by Calibre-Web-Automated
KOSync servers.

Fixes #581

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it helps set the right
context for reviewers.

Did you use AI tools to help write this code? _**PARTIALLY**_ AI
assisted with the esp_http_client migration pattern and base64 encoder.
The root cause analysis (TLS buffer OOM) and solution design were
manual.

---------

Co-authored-by: trilwu <trilwu@users.noreply.github.com>
Co-authored-by: Justin Mitchell <justin@jmitch.com>

lib/KOReaderSync/KOReaderCredentialStore.cpp

@@ -151,16 +151,22 @@ void KOReaderCredentialStore::setServerUrl(const std::string& url) {
 }
 
 std::string KOReaderCredentialStore::getBaseUrl() const {
+  std::string url;
   if (serverUrl.empty()) {
-    return DEFAULT_SERVER_URL;
+    url = DEFAULT_SERVER_URL;
+  } else if (serverUrl.find("://") == std::string::npos) {
+    // Normalize URL: add http:// if no protocol specified (local servers typically don't have SSL)
+    url = "http://" + serverUrl;
+  } else {
+    url = serverUrl;
   }
 
-  // Normalize URL: add http:// if no protocol specified (local servers typically don't have SSL)
-  if (serverUrl.find("://") == std::string::npos) {
-    return "http://" + serverUrl;
+  // Strip trailing slashes to avoid double-slash in API paths
+  while (!url.empty() && url.back() == '/') {
+    url.pop_back();
   }
 
-  return serverUrl;
+  return url;
 }
 
 void KOReaderCredentialStore::setMatchMethod(DocumentMatchMethod method) {

lib/KOReaderSync/KOReaderSyncClient.cpp

@@ -1,102 +1,153 @@
 #include "KOReaderSyncClient.h"
 
 #include <ArduinoJson.h>
-#include <HTTPClient.h>
 #include <Logging.h>
-#include <WiFi.h>
-#include <WiFiClientSecure.h>
+#include <base64.h>
+#include <esp_crt_bundle.h>
+#include <esp_http_client.h>
 
 #include <ctime>
 
 #include "KOReaderCredentialStore.h"
 
+int KOReaderSyncClient::lastHttpCode = 0;
+
 namespace {
 // Device identifier for CrossPoint reader
 constexpr char DEVICE_NAME[] = "CrossPoint";
 constexpr char DEVICE_ID[] = "crosspoint-reader";
 
-void addAuthHeaders(HTTPClient& http) {
-  http.addHeader("Accept", "application/vnd.koreader.v1+json");
-  http.addHeader("x-auth-user", KOREADER_STORE.getUsername().c_str());
-  http.addHeader("x-auth-key", KOREADER_STORE.getMd5Password().c_str());
+// Small TLS buffers to fit in ESP32-C3's limited heap (~46KB free after WiFi).
+// KOSync payloads are tiny JSON (<1KB), so 2KB buffers are sufficient.
+// Default 16KB buffers cause OOM during TLS handshake.
+constexpr int HTTP_BUF_SIZE = 2048;
 
-  // HTTP Basic Auth (RFC 7617) header. This is needed to support koreader sync server embedded in Calibre Web Automated
-  // (https://github.com/crocodilestick/Calibre-Web-Automated/blob/main/cps/progress_syncing/protocols/kosync.py)
-  http.setAuthorization(KOREADER_STORE.getUsername().c_str(), KOREADER_STORE.getPassword().c_str());
+// Response buffer for reading HTTP body
+struct ResponseBuffer {
+  char* data = nullptr;
+  int len = 0;
+  int capacity = 0;
+
+  ~ResponseBuffer() { free(data); }
+
+  bool ensure(int size) {
+    if (size <= capacity) return true;
+    char* newData = (char*)realloc(data, size);
+    if (!newData) return false;
+    data = newData;
+    capacity = size;
+    return true;
+  }
+};
+
+// HTTP event handler to collect response body
+esp_err_t httpEventHandler(esp_http_client_event_t* evt) {
+  auto* buf = static_cast<ResponseBuffer*>(evt->user_data);
+  if (evt->event_id == HTTP_EVENT_ON_DATA && buf) {
+    if (buf->ensure(buf->len + evt->data_len + 1)) {
+      memcpy(buf->data + buf->len, evt->data, evt->data_len);
+      buf->len += evt->data_len;
+      buf->data[buf->len] = '\0';
+    } else {
+      LOG_ERR("KOSync", "Response buffer allocation failed (%d bytes)", evt->data_len);
+    }
+  }
+  return ESP_OK;
 }
 
-bool isHttpsUrl(const std::string& url) { return url.rfind("https://", 0) == 0; }
+// Create configured esp_http_client with small TLS buffers
+esp_http_client_handle_t createClient(const char* url, ResponseBuffer* buf,
+                                      esp_http_client_method_t method = HTTP_METHOD_GET) {
+  esp_http_client_config_t config = {};
+  config.url = url;
+  config.event_handler = httpEventHandler;
+  config.user_data = buf;
+  config.method = method;
+  config.timeout_ms = 15000;
+  config.buffer_size = HTTP_BUF_SIZE;
+  config.buffer_size_tx = HTTP_BUF_SIZE;
+  config.crt_bundle_attach = esp_crt_bundle_attach;
+
+  esp_http_client_handle_t client = esp_http_client_init(&config);
+  if (!client) return nullptr;
+
+  // KOSync auth headers
+  if (esp_http_client_set_header(client, "Accept", "application/vnd.koreader.v1+json") != ESP_OK ||
+      esp_http_client_set_header(client, "x-auth-user", KOREADER_STORE.getUsername().c_str()) != ESP_OK ||
+      esp_http_client_set_header(client, "x-auth-key", KOREADER_STORE.getMd5Password().c_str()) != ESP_OK) {
+    LOG_ERR("KOSync", "Failed to set auth headers");
+    esp_http_client_cleanup(client);
+    return nullptr;
+  }
+
+  // HTTP Basic Auth for Calibre-Web-Automated compatibility
+  std::string credentials = KOREADER_STORE.getUsername() + ":" + KOREADER_STORE.getPassword();
+  String encoded = base64::encode(reinterpret_cast<const uint8_t*>(credentials.data()), credentials.size());
+  std::string authHeader = "Basic " + std::string(encoded.c_str());
+  if (esp_http_client_set_header(client, "Authorization", authHeader.c_str()) != ESP_OK) {
+    LOG_ERR("KOSync", "Failed to set Authorization header");
+    esp_http_client_cleanup(client);
+    return nullptr;
+  }
+
+  return client;
+}
 }  // namespace
 
 KOReaderSyncClient::Error KOReaderSyncClient::authenticate() {
+  lastHttpCode = 0;
   if (!KOREADER_STORE.hasCredentials()) {
     LOG_DBG("KOSync", "No credentials configured");
     return NO_CREDENTIALS;
   }
 
   std::string url = KOREADER_STORE.getBaseUrl() + "/users/auth";
-  LOG_DBG("KOSync", "Authenticating: %s", url.c_str());
+  LOG_DBG("KOSync", "Authenticating: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
 
-  HTTPClient http;
-  std::unique_ptr<WiFiClientSecure> secureClient;
-  WiFiClient plainClient;
+  ResponseBuffer buf;
+  esp_http_client_handle_t client = createClient(url.c_str(), &buf);
+  if (!client) return NETWORK_ERROR;
 
-  if (isHttpsUrl(url)) {
-    secureClient.reset(new WiFiClientSecure);
-    secureClient->setInsecure();
-    http.begin(*secureClient, url.c_str());
-  } else {
-    http.begin(plainClient, url.c_str());
-  }
-  addAuthHeaders(http);
+  esp_err_t err = esp_http_client_perform(client);
+  const int httpCode = esp_http_client_get_status_code(client);
+  lastHttpCode = httpCode;
+  esp_http_client_cleanup(client);
 
-  const int httpCode = http.GET();
-  http.end();
+  LOG_DBG("KOSync", "Auth response: %d (err: %d)", httpCode, err);
 
-  LOG_DBG("KOSync", "Auth response: %d", httpCode);
-
-  if (httpCode == 200) {
-    return OK;
-  } else if (httpCode == 401) {
-    return AUTH_FAILED;
-  } else if (httpCode < 0) {
-    return NETWORK_ERROR;
-  }
+  if (err != ESP_OK) return NETWORK_ERROR;
+  if (httpCode == 200) return OK;
+  if (httpCode == 401) return AUTH_FAILED;
   return SERVER_ERROR;
 }
 
 KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& documentHash,
                                                           KOReaderProgress& outProgress) {
+  lastHttpCode = 0;
   if (!KOREADER_STORE.hasCredentials()) {
     LOG_DBG("KOSync", "No credentials configured");
     return NO_CREDENTIALS;
   }
 
   std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress/" + documentHash;
-  LOG_DBG("KOSync", "Getting progress: %s", url.c_str());
+  LOG_DBG("KOSync", "Getting progress: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
 
-  HTTPClient http;
-  std::unique_ptr<WiFiClientSecure> secureClient;
-  WiFiClient plainClient;
+  ResponseBuffer buf;
+  esp_http_client_handle_t client = createClient(url.c_str(), &buf);
+  if (!client) return NETWORK_ERROR;
 
-  if (isHttpsUrl(url)) {
-    secureClient.reset(new WiFiClientSecure);
-    secureClient->setInsecure();
-    http.begin(*secureClient, url.c_str());
-  } else {
-    http.begin(plainClient, url.c_str());
-  }
-  addAuthHeaders(http);
+  esp_err_t err = esp_http_client_perform(client);
+  const int httpCode = esp_http_client_get_status_code(client);
+  lastHttpCode = httpCode;
+  esp_http_client_cleanup(client);
 
-  const int httpCode = http.GET();
+  LOG_DBG("KOSync", "Get progress response: %d (err: %d)", httpCode, err);
 
-  if (httpCode == 200) {
-    // Parse JSON response from response string
-    String responseBody = http.getString();
-    http.end();
+  if (err != ESP_OK) return NETWORK_ERROR;
 
+  if (httpCode == 200 && buf.data) {
     JsonDocument doc;
-    const DeserializationError error = deserializeJson(doc, responseBody);
+    const DeserializationError error = deserializeJson(doc, buf.data);
 
     if (error) {
       LOG_ERR("KOSync", "JSON parse failed: %s", error.c_str());
@@ -114,44 +165,22 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
     return OK;
   }
 
-  http.end();
-
-  LOG_DBG("KOSync", "Get progress response: %d", httpCode);
-
-  if (httpCode == 401) {
-    return AUTH_FAILED;
-  } else if (httpCode == 404) {
-    return NOT_FOUND;
-  } else if (httpCode < 0) {
-    return NETWORK_ERROR;
-  }
+  if (httpCode == 401) return AUTH_FAILED;
+  if (httpCode == 404) return NOT_FOUND;
   return SERVER_ERROR;
 }
 
 KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgress& progress) {
+  lastHttpCode = 0;
   if (!KOREADER_STORE.hasCredentials()) {
     LOG_DBG("KOSync", "No credentials configured");
     return NO_CREDENTIALS;
   }
 
   std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress";
-  LOG_DBG("KOSync", "Updating progress: %s", url.c_str());
+  LOG_DBG("KOSync", "Updating progress: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
 
-  HTTPClient http;
-  std::unique_ptr<WiFiClientSecure> secureClient;
-  WiFiClient plainClient;
-
-  if (isHttpsUrl(url)) {
-    secureClient.reset(new WiFiClientSecure);
-    secureClient->setInsecure();
-    http.begin(*secureClient, url.c_str());
-  } else {
-    http.begin(plainClient, url.c_str());
-  }
-  addAuthHeaders(http);
-  http.addHeader("Content-Type", "application/json");
-
-  // Build JSON body (timestamp not required per API spec)
+  // Build JSON body
   JsonDocument doc;
   doc["document"] = progress.document;
   doc["progress"] = progress.progress;
@@ -164,18 +193,27 @@ KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgr
 
   LOG_DBG("KOSync", "Request body: %s", body.c_str());
 
-  const int httpCode = http.PUT(body.c_str());
-  http.end();
+  ResponseBuffer buf;
+  esp_http_client_handle_t client = createClient(url.c_str(), &buf, HTTP_METHOD_PUT);
+  if (!client) return NETWORK_ERROR;
 
-  LOG_DBG("KOSync", "Update progress response: %d", httpCode);
-
-  if (httpCode == 200 || httpCode == 202) {
-    return OK;
-  } else if (httpCode == 401) {
-    return AUTH_FAILED;
-  } else if (httpCode < 0) {
+  if (esp_http_client_set_header(client, "Content-Type", "application/json") != ESP_OK ||
+      esp_http_client_set_post_field(client, body.c_str(), body.length()) != ESP_OK) {
+    LOG_ERR("KOSync", "Failed to set request body");
+    esp_http_client_cleanup(client);
     return NETWORK_ERROR;
   }
+
+  esp_err_t err = esp_http_client_perform(client);
+  const int httpCode = esp_http_client_get_status_code(client);
+  lastHttpCode = httpCode;
+  esp_http_client_cleanup(client);
+
+  LOG_DBG("KOSync", "Update progress response: %d (err: %d)", httpCode, err);
+
+  if (err != ESP_OK) return NETWORK_ERROR;
+  if (httpCode == 200 || httpCode == 202) return OK;
+  if (httpCode == 401) return AUTH_FAILED;
   return SERVER_ERROR;
 }
 

lib/KOReaderSync/KOReaderSyncClient.h

@@ -56,4 +56,7 @@ class KOReaderSyncClient {
    * Get human-readable error message.
    */
   static const char* errorString(Error error);
+
+  /** HTTP status code from the last request (for diagnostics). */
+  static int lastHttpCode;
 };