## Summary * **What is the goal of this PR?** Fix KOReader sync failing on HTTPS servers due to TLS out-of-memory on ESP32-C3 * **What changes are included?** - Replace `WiFiClientSecure`/`HTTPClient` with `esp_http_client` (ESP-IDF native API) for all KOSync HTTP requests - Use 2KB TLS buffers instead of the default 16KB — KOSync payloads are tiny JSON (<1KB), so this is more than sufficient - Use `esp_crt_bundle_attach` for proper TLS certificate verification (replaces `setInsecure()`) - Strip trailing slashes from server URL to prevent double-slash in API paths (e.g. `https://server.com//users/auth`) - Add `lastHttpCode` static field for diagnostics - Add free heap logging to help debug memory issues ### Problem The ESP32-C3 has ~46KB free heap after WiFi is initialized. `WiFiClientSecure` allocates 16KB for TLS RX + 16KB for TLS TX = 32KB just for the TLS buffers, leaving almost no room for the actual TLS handshake (which needs additional dynamic allocations). This causes KOReader sync to: 1. Fail silently with network errors on HTTPS servers (including the default `sync.koreader.rocks`) 2. Occasionally crash with heap exhaustion ### Solution `esp_http_client` (the ESP-IDF native HTTP client) allows configuring `buffer_size` and `buffer_size_tx` independently. Setting both to 2KB (total 4KB) leaves plenty of heap for the TLS handshake while still being sufficient for KOSync's small JSON payloads. This also fixes the `setInsecure()` anti-pattern — `esp_crt_bundle_attach` provides proper certificate verification using the ESP-IDF's built-in CA bundle, so credentials and reading history are no longer sent over unverified TLS connections. ### Files changed | File | Change | |------|--------| | `lib/KOReaderSync/KOReaderSyncClient.cpp` | Replace WiFiClientSecure/HTTPClient with esp_http_client; add ResponseBuffer, base64 encoder, createClient helper | | `lib/KOReaderSync/KOReaderSyncClient.h` | Add `lastHttpCode` static field | | `lib/KOReaderSync/KOReaderCredentialStore.cpp` | Strip trailing slashes from base URL | ## Additional Context Tested on a CrossPoint X4 (ESP32-C3 with 4MB flash). Before this change, KOSync auth to `sync.koreader.rocks` (HTTPS) would fail ~80% of the time. After: works reliably. The `base64Encode` helper is needed because `esp_http_client` doesn't have a built-in `setAuthorization()` method like `HTTPClient` does. This is used for the HTTP Basic Auth header required by Calibre-Web-Automated KOSync servers. Fixes #581 --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**PARTIALLY**_ AI assisted with the esp_http_client migration pattern and base64 encoder. The root cause analysis (TLS buffer OOM) and solution design were manual. --------- Co-authored-by: trilwu <trilwu@users.noreply.github.com> Co-authored-by: Justin Mitchell <justin@jmitch.com>
176 lines
4.8 KiB
C++
176 lines
4.8 KiB
C++
#include "KOReaderCredentialStore.h"
|
|
|
|
#include <HalStorage.h>
|
|
#include <Logging.h>
|
|
#include <MD5Builder.h>
|
|
#include <ObfuscationUtils.h>
|
|
#include <Serialization.h>
|
|
|
|
#include "../../src/JsonSettingsIO.h"
|
|
|
|
// Initialize the static instance
|
|
KOReaderCredentialStore KOReaderCredentialStore::instance;
|
|
|
|
namespace {
|
|
// File format version (for binary migration)
|
|
constexpr uint8_t KOREADER_FILE_VERSION = 1;
|
|
|
|
// File paths
|
|
constexpr char KOREADER_FILE_BIN[] = "/.crosspoint/koreader.bin";
|
|
constexpr char KOREADER_FILE_JSON[] = "/.crosspoint/koreader.json";
|
|
constexpr char KOREADER_FILE_BAK[] = "/.crosspoint/koreader.bin.bak";
|
|
|
|
// Default sync server URL
|
|
constexpr char DEFAULT_SERVER_URL[] = "https://sync.koreader.rocks:443";
|
|
|
|
// Legacy obfuscation key - "KOReader" in ASCII (only used for binary migration)
|
|
constexpr uint8_t LEGACY_OBFUSCATION_KEY[] = {0x4B, 0x4F, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72};
|
|
constexpr size_t LEGACY_KEY_LENGTH = sizeof(LEGACY_OBFUSCATION_KEY);
|
|
|
|
void legacyDeobfuscate(std::string& data) {
|
|
for (size_t i = 0; i < data.size(); i++) {
|
|
data[i] ^= LEGACY_OBFUSCATION_KEY[i % LEGACY_KEY_LENGTH];
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
bool KOReaderCredentialStore::saveToFile() const {
|
|
Storage.mkdir("/.crosspoint");
|
|
return JsonSettingsIO::saveKOReader(*this, KOREADER_FILE_JSON);
|
|
}
|
|
|
|
bool KOReaderCredentialStore::loadFromFile() {
|
|
// Try JSON first
|
|
if (Storage.exists(KOREADER_FILE_JSON)) {
|
|
String json = Storage.readFile(KOREADER_FILE_JSON);
|
|
if (!json.isEmpty()) {
|
|
bool resave = false;
|
|
bool result = JsonSettingsIO::loadKOReader(*this, json.c_str(), &resave);
|
|
if (result && resave) {
|
|
saveToFile();
|
|
LOG_DBG("KRS", "Resaved KOReader credentials to update format");
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// Fall back to binary migration
|
|
if (Storage.exists(KOREADER_FILE_BIN)) {
|
|
if (loadFromBinaryFile()) {
|
|
if (saveToFile()) {
|
|
Storage.rename(KOREADER_FILE_BIN, KOREADER_FILE_BAK);
|
|
LOG_DBG("KRS", "Migrated koreader.bin to koreader.json");
|
|
return true;
|
|
} else {
|
|
LOG_ERR("KRS", "Failed to save KOReader credentials during migration");
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
LOG_DBG("KRS", "No credentials file found");
|
|
return false;
|
|
}
|
|
|
|
bool KOReaderCredentialStore::loadFromBinaryFile() {
|
|
FsFile file;
|
|
if (!Storage.openFileForRead("KRS", KOREADER_FILE_BIN, file)) {
|
|
return false;
|
|
}
|
|
|
|
uint8_t version;
|
|
serialization::readPod(file, version);
|
|
if (version != KOREADER_FILE_VERSION) {
|
|
LOG_DBG("KRS", "Unknown file version: %u", version);
|
|
return false;
|
|
}
|
|
|
|
if (file.available()) {
|
|
serialization::readString(file, username);
|
|
} else {
|
|
username.clear();
|
|
}
|
|
|
|
if (file.available()) {
|
|
serialization::readString(file, password);
|
|
legacyDeobfuscate(password);
|
|
} else {
|
|
password.clear();
|
|
}
|
|
|
|
if (file.available()) {
|
|
serialization::readString(file, serverUrl);
|
|
} else {
|
|
serverUrl.clear();
|
|
}
|
|
|
|
if (file.available()) {
|
|
uint8_t method;
|
|
serialization::readPod(file, method);
|
|
matchMethod = static_cast<DocumentMatchMethod>(method);
|
|
} else {
|
|
matchMethod = DocumentMatchMethod::FILENAME;
|
|
}
|
|
|
|
LOG_DBG("KRS", "Loaded KOReader credentials from binary for user: %s", username.c_str());
|
|
return true;
|
|
}
|
|
|
|
void KOReaderCredentialStore::setCredentials(const std::string& user, const std::string& pass) {
|
|
username = user;
|
|
password = pass;
|
|
LOG_DBG("KRS", "Set credentials for user: %s", user.c_str());
|
|
}
|
|
|
|
std::string KOReaderCredentialStore::getMd5Password() const {
|
|
if (password.empty()) {
|
|
return "";
|
|
}
|
|
|
|
// Calculate MD5 hash of password using ESP32's MD5Builder
|
|
MD5Builder md5;
|
|
md5.begin();
|
|
md5.add(password.c_str());
|
|
md5.calculate();
|
|
|
|
return md5.toString().c_str();
|
|
}
|
|
|
|
bool KOReaderCredentialStore::hasCredentials() const { return !username.empty() && !password.empty(); }
|
|
|
|
void KOReaderCredentialStore::clearCredentials() {
|
|
username.clear();
|
|
password.clear();
|
|
saveToFile();
|
|
LOG_DBG("KRS", "Cleared KOReader credentials");
|
|
}
|
|
|
|
void KOReaderCredentialStore::setServerUrl(const std::string& url) {
|
|
serverUrl = url;
|
|
LOG_DBG("KRS", "Set server URL: %s", url.empty() ? "(default)" : url.c_str());
|
|
}
|
|
|
|
std::string KOReaderCredentialStore::getBaseUrl() const {
|
|
std::string url;
|
|
if (serverUrl.empty()) {
|
|
url = DEFAULT_SERVER_URL;
|
|
} else if (serverUrl.find("://") == std::string::npos) {
|
|
// Normalize URL: add http:// if no protocol specified (local servers typically don't have SSL)
|
|
url = "http://" + serverUrl;
|
|
} else {
|
|
url = serverUrl;
|
|
}
|
|
|
|
// Strip trailing slashes to avoid double-slash in API paths
|
|
while (!url.empty() && url.back() == '/') {
|
|
url.pop_back();
|
|
}
|
|
|
|
return url;
|
|
}
|
|
|
|
void KOReaderCredentialStore::setMatchMethod(DocumentMatchMethod method) {
|
|
matchMethod = method;
|
|
LOG_DBG("KRS", "Set match method: %s", method == DocumentMatchMethod::FILENAME ? "Filename" : "Binary");
|
|
}
|