fix: oom exceptions for OPDS, KOSync, and OTA via wolfssl (#2475)

This commit is contained in:
Justin Mitchell
2026-07-12 01:53:26 +03:00
committed by GitHub
parent 1f5669a08a
commit 3e627112f6
13 changed files with 413 additions and 233 deletions
+77 -132
View File
@@ -2,10 +2,10 @@
#include <ArduinoJson.h>
#include <Logging.h>
#include <esp_crt_bundle.h>
#include <esp_http_client.h>
#include <SecureHttpClient.h>
#include <base64.h>
#include <ctime>
#include <string>
#include "KOReaderCredentialStore.h"
@@ -16,82 +16,35 @@ namespace {
constexpr char DEVICE_NAME[] = "CrossPoint";
constexpr char DEVICE_ID[] = "crosspoint-reader";
// Small TLS buffers to fit in ESP32-C3's limited heap (~46KB free after WiFi).
// KOSync payloads are tiny JSON (<1KB), so 2KB buffers are sufficient.
// Default 16KB buffers cause OOM during TLS handshake.
constexpr int HTTP_BUF_SIZE = 2048;
// Cloudflare tunnels send a 3-cert Google Trust Services chain. During the TLS handshake
// mbedTLS makes many small allocations that collectively consume ~48KB of heap. With only
// ~50KB free after WiFi connects, the session drove min-free-ever down to 2600 bytes before
// failing with MBEDTLS_ERR_X509_ALLOC_FAILED (-0x2880). Check total free heap (not max
// contiguous block) because the failure mode is aggregate exhaustion, not one large alloc.
// KOSync's TLS-1.3 servers can't be reached through the precompiled system
// mbedTLS (TLS 1.3 is stubbed out), so requests run over wolfSSL via
// SecureHttpClient. The handshake still needs working heap; gate on it. wolfSSL's
// footprint is smaller than mbedTLS's old ~48KB peak, but keep a conservative
// floor. Check both total free heap and largest contiguous block so fragmented
// heap does not fall through into a failed TLS allocation path.
constexpr uint32_t MIN_HEAP_FOR_TLS = 55000;
// Response buffer for reading HTTP body
struct ResponseBuffer {
char* data = nullptr;
int len = 0;
int capacity = 0;
~ResponseBuffer() { free(data); }
bool ensure(int size) {
if (size <= capacity) return true;
char* newData = (char*)realloc(data, size);
if (!newData) return false;
data = newData;
capacity = size;
return true;
}
};
// HTTP event handler to collect response body
esp_err_t httpEventHandler(esp_http_client_event_t* evt) {
auto* buf = static_cast<ResponseBuffer*>(evt->user_data);
if (evt->event_id == HTTP_EVENT_ON_DATA && buf) {
if (buf->ensure(buf->len + evt->data_len + 1)) {
memcpy(buf->data + buf->len, evt->data, evt->data_len);
buf->len += evt->data_len;
buf->data[buf->len] = '\0';
} else {
LOG_ERR("KOSync", "Response buffer allocation failed (%d bytes)", evt->data_len);
}
}
return ESP_OK;
// Apply the shared KOSync auth headers after begin(). x-auth-* is the native
// KOSync scheme; Basic auth is added for Calibre-Web-Automated compatibility.
void applyAuthHeaders(freeink::SecureHttpClient& http) {
http.addHeader("Accept", "application/vnd.koreader.v1+json");
http.addHeader("x-auth-user", KOREADER_STORE.getUsername());
http.addHeader("x-auth-key", KOREADER_STORE.getMd5Password());
const std::string credentials = KOREADER_STORE.getUsername() + ":" + KOREADER_STORE.getPassword();
const String encoded = base64::encode(credentials.c_str());
http.addHeader("Authorization", std::string("Basic ") + encoded.c_str());
}
// Create configured esp_http_client with small TLS buffers
esp_http_client_handle_t createClient(const char* url, ResponseBuffer* buf,
esp_http_client_method_t method = HTTP_METHOD_GET) {
esp_http_client_config_t config = {};
config.url = url;
config.event_handler = httpEventHandler;
config.user_data = buf;
config.method = method;
config.timeout_ms = 15000;
config.buffer_size = HTTP_BUF_SIZE;
config.buffer_size_tx = HTTP_BUF_SIZE;
config.crt_bundle_attach = esp_crt_bundle_attach;
// HTTP Basic Auth for Calibre-Web-Automated compatibility
config.username = KOREADER_STORE.getUsername().c_str();
config.password = KOREADER_STORE.getPassword().c_str();
config.auth_type = HTTP_AUTH_TYPE_BASIC;
esp_http_client_handle_t client = esp_http_client_init(&config);
if (!client) return nullptr;
// KOSync auth headers
if (esp_http_client_set_header(client, "Accept", "application/vnd.koreader.v1+json") != ESP_OK ||
esp_http_client_set_header(client, "x-auth-user", KOREADER_STORE.getUsername().c_str()) != ESP_OK ||
esp_http_client_set_header(client, "x-auth-key", KOREADER_STORE.getMd5Password().c_str()) != ESP_OK) {
LOG_ERR("KOSync", "Failed to set auth headers");
esp_http_client_cleanup(client);
return nullptr;
// True when free heap is too low to risk a TLS handshake.
bool insufficientHeap() {
const uint32_t freeHeap = ESP.getFreeHeap();
const uint32_t maxAllocHeap = ESP.getMaxAllocHeap();
if (freeHeap < MIN_HEAP_FOR_TLS || maxAllocHeap < MIN_HEAP_FOR_TLS) {
LOG_ERR("KOSync", "Insufficient heap for TLS handshake: %u bytes free, %u max alloc (need %u)", freeHeap,
maxAllocHeap, MIN_HEAP_FOR_TLS);
return true;
}
return client;
return false;
}
} // namespace
@@ -102,26 +55,24 @@ KOReaderSyncClient::Error KOReaderSyncClient::authenticate() {
return NO_CREDENTIALS;
}
std::string url = KOREADER_STORE.getBaseUrl() + "/users/auth";
const uint32_t freeHeap = ESP.getFreeHeap();
LOG_DBG("KOSync", "Authenticating: %s (heap: %u)", url.c_str(), (unsigned)freeHeap);
if (freeHeap < MIN_HEAP_FOR_TLS) {
LOG_ERR("KOSync", "Insufficient heap for TLS handshake: %u bytes free (need %u)", freeHeap, MIN_HEAP_FOR_TLS);
return LOW_MEMORY;
const std::string url = KOREADER_STORE.getBaseUrl() + "/users/auth";
LOG_DBG("KOSync", "Authenticating: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
if (insufficientHeap()) return LOW_MEMORY;
freeink::SecureHttpClient http;
http.setInsecure();
if (!http.begin(url)) {
LOG_ERR("KOSync", "Bad URL: %s", url.c_str());
return NETWORK_ERROR;
}
ResponseBuffer buf;
esp_http_client_handle_t client = createClient(url.c_str(), &buf);
if (!client) return NETWORK_ERROR;
esp_err_t err = esp_http_client_perform(client);
const int httpCode = esp_http_client_get_status_code(client);
applyAuthHeaders(http);
const int httpCode = http.GET();
http.end();
lastHttpCode = httpCode;
esp_http_client_cleanup(client);
LOG_DBG("KOSync", "Auth response: %d (err: %d)", httpCode, err);
LOG_DBG("KOSync", "Auth response: %d", httpCode);
if (err != ESP_OK) return NETWORK_ERROR;
if (httpCode <= 0) return NETWORK_ERROR;
if (httpCode == 200) return OK;
if (httpCode == 401) return AUTH_FAILED;
return SERVER_ERROR;
@@ -135,30 +86,31 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
return NO_CREDENTIALS;
}
std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress/" + documentHash;
const uint32_t freeHeap = ESP.getFreeHeap();
LOG_DBG("KOSync", "Getting progress: %s (heap: %u)", url.c_str(), (unsigned)freeHeap);
if (freeHeap < MIN_HEAP_FOR_TLS) {
LOG_ERR("KOSync", "Insufficient heap for TLS handshake: %u bytes free (need %u)", freeHeap, MIN_HEAP_FOR_TLS);
return LOW_MEMORY;
const std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress/" + documentHash;
LOG_DBG("KOSync", "Getting progress: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
if (insufficientHeap()) return LOW_MEMORY;
freeink::SecureHttpClient http;
http.setInsecure();
if (!http.begin(url)) {
LOG_ERR("KOSync", "Bad URL: %s", url.c_str());
return NETWORK_ERROR;
}
applyAuthHeaders(http);
const int httpCode = http.GET();
lastHttpCode = httpCode;
LOG_DBG("KOSync", "Get progress response: %d", httpCode);
if (httpCode <= 0) {
http.end();
return NETWORK_ERROR;
}
ResponseBuffer buf;
esp_http_client_handle_t client = createClient(url.c_str(), &buf);
if (!client) return NETWORK_ERROR;
esp_err_t err = esp_http_client_perform(client);
const int httpCode = esp_http_client_get_status_code(client);
lastHttpCode = httpCode;
esp_http_client_cleanup(client);
LOG_DBG("KOSync", "Get progress response: %d (err: %d)", httpCode, err);
if (err != ESP_OK) return NETWORK_ERROR;
if (httpCode == 200 && buf.data) {
if (httpCode == 200) {
JsonDocument doc;
const DeserializationError error = deserializeJson(doc, buf.data);
const DeserializationError error = deserializeJson(doc, http.getString().c_str());
http.end();
if (error) {
LOG_ERR("KOSync", "JSON parse failed: %s", error.c_str());
@@ -176,6 +128,7 @@ KOReaderSyncClient::Error KOReaderSyncClient::getProgress(const std::string& doc
return OK;
}
http.end();
if (httpCode == 401) return AUTH_FAILED;
if (httpCode == 404) return NOT_FOUND;
return SERVER_ERROR;
@@ -188,13 +141,9 @@ KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgr
return NO_CREDENTIALS;
}
std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress";
const uint32_t freeHeap = ESP.getFreeHeap();
LOG_DBG("KOSync", "Updating progress: %s (heap: %u)", url.c_str(), (unsigned)freeHeap);
if (freeHeap < MIN_HEAP_FOR_TLS) {
LOG_ERR("KOSync", "Insufficient heap for TLS handshake: %u bytes free (need %u)", freeHeap, MIN_HEAP_FOR_TLS);
return LOW_MEMORY;
}
const std::string url = KOREADER_STORE.getBaseUrl() + "/syncs/progress";
LOG_DBG("KOSync", "Updating progress: %s (heap: %u)", url.c_str(), (unsigned)ESP.getFreeHeap());
if (insufficientHeap()) return LOW_MEMORY;
// Build JSON body
JsonDocument doc;
@@ -209,25 +158,21 @@ KOReaderSyncClient::Error KOReaderSyncClient::updateProgress(const KOReaderProgr
LOG_DBG("KOSync", "Request body: %s", body.c_str());
ResponseBuffer buf;
esp_http_client_handle_t client = createClient(url.c_str(), &buf, HTTP_METHOD_PUT);
if (!client) return NETWORK_ERROR;
if (esp_http_client_set_header(client, "Content-Type", "application/json") != ESP_OK ||
esp_http_client_set_post_field(client, body.c_str(), body.length()) != ESP_OK) {
LOG_ERR("KOSync", "Failed to set request body");
esp_http_client_cleanup(client);
freeink::SecureHttpClient http;
http.setInsecure();
if (!http.begin(url)) {
LOG_ERR("KOSync", "Bad URL: %s", url.c_str());
return NETWORK_ERROR;
}
esp_err_t err = esp_http_client_perform(client);
const int httpCode = esp_http_client_get_status_code(client);
applyAuthHeaders(http);
http.addHeader("Content-Type", "application/json");
const int httpCode = http.sendRequest("PUT", body);
http.end();
lastHttpCode = httpCode;
esp_http_client_cleanup(client);
LOG_DBG("KOSync", "Update progress response: %d (err: %d)", httpCode, err);
LOG_DBG("KOSync", "Update progress response: %d", httpCode);
if (err != ESP_OK) return NETWORK_ERROR;
if (httpCode <= 0) return NETWORK_ERROR;
if (httpCode == 200 || httpCode == 202) return OK;
if (httpCode == 401) return AUTH_FAILED;
return SERVER_ERROR;
+55 -18
View File
@@ -5,6 +5,17 @@
#include <cstring>
namespace {
constexpr size_t ENTRY_STORAGE_CAPACITY = 64;
constexpr size_t MAX_ENTRIES = ENTRY_STORAGE_CAPACITY - 2;
constexpr size_t MAX_TITLE_CHARS = 160;
constexpr size_t MAX_AUTHOR_CHARS = 120;
constexpr size_t MAX_ID_CHARS = 128;
constexpr size_t MAX_HREF_CHARS = 768;
constexpr size_t MAX_SEARCH_TEMPLATE_CHARS = 768;
constexpr size_t MAX_PAGE_URL_CHARS = 768;
} // namespace
OpdsParser::OpdsParser() {
parser = XML_ParserCreate(nullptr);
if (!parser) {
@@ -12,6 +23,7 @@ OpdsParser::OpdsParser() {
LOG_DBG("OPDS", "Couldn't allocate memory for parser");
return;
}
entries.reserve(ENTRY_STORAGE_CAPACITY);
XML_SetUserData(parser, this);
XML_SetElementHandler(parser, startElement, endElement);
XML_SetCharacterDataHandler(parser, characterData);
@@ -71,6 +83,8 @@ void OpdsParser::clear() {
currentEntry = OpdsEntry{};
currentText.clear();
inEntry = inTitle = inAuthor = inAuthorName = inId = false;
collectCurrentEntry = false;
feedTruncated = false;
}
std::vector<OpdsEntry> OpdsParser::getBooks() const {
@@ -88,9 +102,33 @@ const char* OpdsParser::findAttribute(const XML_Char** atts, const char* name) {
return nullptr;
}
void OpdsParser::assignBounded(std::string& target, const char* value, const size_t maxLen) {
if (!value) {
target.clear();
return;
}
target.assign(value, strnlen(value, maxLen));
}
void OpdsParser::appendBounded(std::string& target, const char* value, const size_t len, const size_t maxLen) {
if (target.size() >= maxLen) return;
const size_t remaining = maxLen - target.size();
target.append(value, len < remaining ? len : remaining);
}
void XMLCALL OpdsParser::startElement(void* userData, const XML_Char* name, const XML_Char** atts) {
auto* self = static_cast<OpdsParser*>(userData);
if (strcmp(name, "entry") == 0 || strstr(name, ":entry") != nullptr) {
self->inEntry = true;
self->collectCurrentEntry = self->entries.size() < MAX_ENTRIES;
self->feedTruncated = self->feedTruncated || !self->collectCurrentEntry;
self->currentEntry = OpdsEntry{};
self->currentText.clear();
self->inTitle = self->inAuthor = self->inAuthorName = self->inId = false;
return;
}
if (strcmp(name, "link") == 0 || strstr(name, ":link") != nullptr) {
const char* href = findAttribute(atts, "href");
if (href) {
@@ -98,17 +136,16 @@ void XMLCALL OpdsParser::startElement(void* userData, const XML_Char* name, cons
const char* type = findAttribute(atts, "type");
if (rel && strcmp(rel, "search") == 0) {
std::string sHref(href);
if (sHref.find("{searchTerms}") != std::string::npos) {
self->searchTemplate = sHref;
if (strstr(href, "{searchTerms}") != nullptr) {
assignBounded(self->searchTemplate, href, MAX_SEARCH_TEMPLATE_CHARS);
}
} else if (rel && strcmp(rel, "next") == 0 && !self->inEntry) {
self->nextPageUrl = href;
assignBounded(self->nextPageUrl, href, MAX_PAGE_URL_CHARS);
} else if (rel && strcmp(rel, "previous") == 0 && !self->inEntry) {
self->prevPageUrl = href;
assignBounded(self->prevPageUrl, href, MAX_PAGE_URL_CHARS);
}
if (self->inEntry) {
if (self->inEntry && self->collectCurrentEntry) {
if (rel && type && strstr(rel, "opds-spec.org/acquisition") != nullptr &&
strcmp(type, "application/epub+zip") == 0) {
// Prefer plain EPUB links over derived formats when multiple
@@ -119,25 +156,19 @@ void XMLCALL OpdsParser::startElement(void* userData, const XML_Char* name, cons
self->currentEntry.href.find("/epub/") != std::string::npos);
if (self->currentEntry.type != OpdsEntryType::BOOK || (isPlainEpub && !alreadyHasPlainEpub)) {
self->currentEntry.type = OpdsEntryType::BOOK;
self->currentEntry.href = href;
assignBounded(self->currentEntry.href, href, MAX_HREF_CHARS);
}
} else if (type && strstr(type, "application/atom+xml") != nullptr) {
if (self->currentEntry.type != OpdsEntryType::BOOK) {
self->currentEntry.type = OpdsEntryType::NAVIGATION;
self->currentEntry.href = href;
assignBounded(self->currentEntry.href, href, MAX_HREF_CHARS);
}
}
}
}
}
if (strcmp(name, "entry") == 0 || strstr(name, ":entry") != nullptr) {
self->inEntry = true;
self->currentEntry = OpdsEntry{};
return;
}
if (!self->inEntry) return;
if (!self->inEntry || !self->collectCurrentEntry) return;
if (strcmp(name, "title") == 0 || strstr(name, ":title") != nullptr) {
self->inTitle = true;
@@ -157,10 +188,11 @@ void XMLCALL OpdsParser::endElement(void* userData, const XML_Char* name) {
auto* self = static_cast<OpdsParser*>(userData);
if (strcmp(name, "entry") == 0 || strstr(name, ":entry") != nullptr) {
if (!self->currentEntry.title.empty() && !self->currentEntry.href.empty()) {
if (self->collectCurrentEntry && !self->currentEntry.title.empty() && !self->currentEntry.href.empty()) {
self->entries.push_back(self->currentEntry);
}
self->inEntry = false;
self->collectCurrentEntry = false;
} else if (self->inEntry) {
if (strcmp(name, "title") == 0 || strstr(name, ":title") != nullptr) {
if (self->inTitle) self->currentEntry.title = self->currentText;
@@ -179,7 +211,12 @@ void XMLCALL OpdsParser::endElement(void* userData, const XML_Char* name) {
void XMLCALL OpdsParser::characterData(void* userData, const XML_Char* s, const int len) {
auto* self = static_cast<OpdsParser*>(userData);
if (self->inTitle || self->inAuthorName || self->inId) {
self->currentText.append(s, len);
if (!self->collectCurrentEntry) return;
if (self->inTitle) {
appendBounded(self->currentText, s, len, MAX_TITLE_CHARS);
} else if (self->inAuthorName) {
appendBounded(self->currentText, s, len, MAX_AUTHOR_CHARS);
} else if (self->inId) {
appendBounded(self->currentText, s, len, MAX_ID_CHARS);
}
}
+5
View File
@@ -61,6 +61,7 @@ class OpdsParser final : public Print {
void flush() override;
bool error() const;
bool truncated() const { return feedTruncated; }
operator bool() { return !error(); }
@@ -93,6 +94,8 @@ class OpdsParser final : public Print {
std::string prevPageUrl;
// Helper to find attribute value
static const char* findAttribute(const XML_Char** atts, const char* name);
static void assignBounded(std::string& target, const char* value, size_t maxLen);
static void appendBounded(std::string& target, const char* value, size_t len, size_t maxLen);
XML_Parser parser = nullptr;
std::vector<OpdsEntry> entries;
@@ -105,6 +108,8 @@ class OpdsParser final : public Print {
bool inAuthor = false;
bool inAuthorName = false;
bool inId = false;
bool collectCurrentEntry = false;
bool errorOccured = false;
bool feedTruncated = false;
};
+19
View File
@@ -37,6 +37,19 @@ build_flags =
-DPNG_MAX_BUFFERED_PIXELS=16416
-DFREEINK_DEVICE_X4=1
-DFREEINK_DEVICE_X3=1
-DFREEINK_NET_WOLFSSL=1
-DWOLFSSL_USER_SETTINGS
-DWOLFSSL_OPTIONS_H
-DWOLFSSL_CLIENT_EXAMPLE
-DWOLFSSL_TLS13
-DWOLFSSL_SP_RISCV32
-DHAVE_TLS_EXTENSIONS
-DHAVE_SUPPORTED_CURVES
-DHAVE_HKDF
-DHAVE_FFDHE_2048
-DHAVE_CURVE25519
-DWC_RSA_PSS
-DHAVE_SNI
-Wno-bidi-chars
-Wl,--wrap=panic_print_backtrace,--wrap=panic_abort,--wrap=bootloader_common_check_efuse_blk_validity
-fno-exceptions
@@ -51,6 +64,7 @@ board_build.flash_size = 16MB
board_build.partitions = partitions.csv
extra_scripts =
pre:scripts/patch_wolfssl.py
pre:scripts/build_html.py
pre:scripts/gen_i18n.py
pre:scripts/git_branch.py
@@ -65,6 +79,7 @@ lib_deps =
SDCardManager=symlink://freeink-sdk/libs/hardware/SDCardManager
BoardConfig=symlink://freeink-sdk/libs/hardware/BoardConfig
PowerManager=symlink://freeink-sdk/libs/hardware/PowerManager
SecureNet=symlink://freeink-sdk/libs/network/SecureNet
FreeInkUI=symlink://freeink-sdk/libs/ui/FreeInkUI
Icons=symlink://freeink-sdk/libs/assets/Icons
bblanchon/ArduinoJson @ 7.4.2
@@ -72,6 +87,10 @@ lib_deps =
bitbank2/PNGdec @ 1.1.6
https://github.com/bitbank2/JPEGDEC.git#86282979224c8a32fd51e091ed5a35b0c699a52b
links2004/WebSockets @ 2.7.3
wolfssl/Arduino-wolfSSL @ 5.7.2
lib_ignore =
BLE
[env:default]
extends = base
+29
View File
@@ -0,0 +1,29 @@
from pathlib import Path
Import("env")
PROJECT_DIR = Path(env.subst("$PROJECT_DIR"))
MARKER = "/* CrossPoint wolfSSL compatibility overrides */"
OVERRIDES = f"""
{MARKER}
#undef NO_DH
#ifndef HAVE_FFDHE_2048
#define HAVE_FFDHE_2048
#endif
#undef FP_MAX_BITS
#define FP_MAX_BITS 16384
"""
def patch_user_settings(path: Path) -> None:
text = path.read_text()
if MARKER in text:
text = text.split(MARKER, 1)[0].rstrip()
path.write_text(text + OVERRIDES + "\n")
print(f"Patched wolfSSL settings: {path.relative_to(PROJECT_DIR)}")
for settings in PROJECT_DIR.glob(".pio/libdeps/*/Arduino-wolfSSL/src/user_settings.h"):
patch_user_settings(settings)
@@ -1,5 +1,6 @@
#include "OpdsBookBrowserActivity.h"
#include <Arduino.h>
#include <GfxRenderer.h>
#include <I18n.h>
#include <Logging.h>
@@ -19,7 +20,9 @@
namespace {
constexpr int PAGE_ITEMS = 23;
}
constexpr int DOWNLOAD_PROGRESS_STEP_PERCENT = 5;
constexpr unsigned long DOWNLOAD_PROGRESS_MIN_UPDATE_MS = 5000;
} // namespace
void OpdsBookBrowserActivity::onEnter() {
Activity::onEnter();
@@ -193,7 +196,7 @@ void OpdsBookBrowserActivity::fetchFeed(const std::string& path) {
return;
}
std::string url = (path.find("http") == 0) ? path : UrlUtils::buildUrl(server.url, path);
std::string url = UrlUtils::buildUrl(server.url, path);
LOG_DBG("OPDS", "Fetching: %s", url.c_str());
OpdsParser parser;
{
@@ -216,14 +219,19 @@ void OpdsBookBrowserActivity::fetchFeed(const std::string& path) {
searchTemplate = parser.getSearchTemplate();
const auto& nextUrl = parser.getNextPageUrl();
const auto& prevUrl = parser.getPrevPageUrl();
const bool feedTruncated = parser.truncated();
entries = std::move(parser).getEntries();
entries.reserve(entries.size() + (prevUrl.empty() ? 0 : 1) + (nextUrl.empty() ? 0 : 1));
if (!prevUrl.empty()) {
entries.insert(entries.begin(), OpdsEntry{OpdsEntryType::NAVIGATION, tr(STR_PREV_PAGE), "", prevUrl, ""});
}
if (!nextUrl.empty()) {
entries.push_back(OpdsEntry{OpdsEntryType::NAVIGATION, tr(STR_NEXT_PAGE), "", nextUrl, ""});
}
if (feedTruncated) {
LOG_INF("OPDS", "Feed truncated to fit memory");
}
selectorIndex = 0;
state = entries.empty() ? BrowserState::ERROR : BrowserState::BROWSING;
@@ -231,6 +239,8 @@ void OpdsBookBrowserActivity::fetchFeed(const std::string& path) {
requestUpdate();
}
void OpdsBookBrowserActivity::releaseEntries() { std::vector<OpdsEntry>().swap(entries); }
void OpdsBookBrowserActivity::navigateToEntry(const OpdsEntry& entry) {
navigationHistory.push_back(currentPath);
// Resolve to a full URL so sub-sub-navigation retains parent path context
@@ -239,7 +249,7 @@ void OpdsBookBrowserActivity::navigateToEntry(const OpdsEntry& entry) {
state = BrowserState::LOADING;
statusMessage = tr(STR_LOADING);
entries.clear();
releaseEntries();
selectorIndex = 0;
requestUpdate(true);
fetchFeed(currentPath);
@@ -253,7 +263,7 @@ void OpdsBookBrowserActivity::navigateBack() {
navigationHistory.pop_back();
state = BrowserState::LOADING;
statusMessage = tr(STR_LOADING);
entries.clear();
releaseEntries();
selectorIndex = 0;
requestUpdate();
fetchFeed(currentPath);
@@ -273,12 +283,22 @@ void OpdsBookBrowserActivity::downloadBook(const OpdsEntry& book) {
"/" + StringUtils::sanitizeFilename((book.author.empty() ? "" : book.author + " - ") + book.title) + ".epub";
LOG_DBG("OPDS", "Downloading: %s -> %s", downloadUrl.c_str(), filename.c_str());
int lastRenderedPercent = -1;
unsigned long lastProgressUpdateMs = 0;
const auto result = HttpDownloader::downloadToFile(
downloadUrl, filename,
[this](const size_t downloaded, const size_t total) {
[this, &lastRenderedPercent, &lastProgressUpdateMs](const size_t downloaded, const size_t total) {
downloadProgress = downloaded;
downloadTotal = total;
const int percent = total > 0 ? static_cast<int>(static_cast<uint64_t>(downloaded) * 100 / total) : 0;
const unsigned long now = millis();
if (percent >= 100 || lastRenderedPercent < 0 ||
percent >= lastRenderedPercent + DOWNLOAD_PROGRESS_STEP_PERCENT ||
now - lastProgressUpdateMs >= DOWNLOAD_PROGRESS_MIN_UPDATE_MS) {
lastRenderedPercent = percent;
lastProgressUpdateMs = now;
requestUpdate(true);
}
},
nullptr, server.username, server.password);
@@ -286,6 +306,7 @@ void OpdsBookBrowserActivity::downloadBook(const OpdsEntry& book) {
clearBookCache(filename);
state = BrowserState::BROWSING;
} else {
LOG_ERR("OPDS", "Download failed: %d", static_cast<int>(result));
state = BrowserState::ERROR;
errorMessage = tr(STR_DOWNLOAD_FAILED);
}
@@ -340,6 +361,8 @@ void OpdsBookBrowserActivity::performSearch(const std::string& query) {
state = BrowserState::LOADING;
statusMessage = tr(STR_LOADING);
releaseEntries();
selectorIndex = 0;
requestUpdate(true);
fetchFeed(url);
}
@@ -46,6 +46,7 @@ class OpdsBookBrowserActivity final : public Activity {
void launchWifiSelection();
void onWifiSelectionComplete(bool connected);
void fetchFeed(const std::string& path);
void releaseEntries();
void navigateToEntry(const OpdsEntry& entry);
void navigateBack();
void downloadBook(const OpdsEntry& book);
+100 -16
View File
@@ -4,27 +4,34 @@
#include <Logging.h>
#include <Memory.h>
#include <base64.h>
#include <esp_crt_bundle.h>
#include <esp_http_client.h>
#include <cstring>
#include <functional>
#include <string>
#if defined(FREEINK_NET_WOLFSSL)
#include <SecureHttpClient.h>
extern "C" void wolfSSL_Arduino_Serial_Print(const char* const msg) { LOG_DBG("WOLFSSL", "%s", msg); }
#else
#include <esp_crt_bundle.h>
#include <esp_http_client.h>
#endif
namespace {
// RX holds the response headers. 4096 fits real OPDS servers; GitHub's release
// CDN sends more and logs HTTP_HEADER "Buffer length is small", but that's
// non-fatal: the headers we read (Location, Content-Length) come first and
// survive. Smaller keeps contiguous heap free while WiFi and TLS are up. TX
// only carries our GET; the body streams in READ_CHUNK pieces.
constexpr int HTTP_RX_BUF = 4096;
constexpr int HTTP_TX_BUF = 1024;
#if !defined(FREEINK_NET_WOLFSSL)
// RX holds the response headers. Smaller buffers leave enough contiguous heap
// for mbedTLS on redirect-heavy OPDS feeds while still preserving the headers
// we read directly (Location, Content-Length).
constexpr int HTTP_RX_BUF = 2048;
constexpr int HTTP_TX_BUF = 512;
#endif
// Per-socket-op timeout. Some OPDS download endpoints are slow to send headers
// (>15s) and chunked catalogs stall mid-body, so 15s killed them. 60s gives
// slow servers room. esp_http_client's timeout_ms is uint32, so unlike Arduino
// HTTPClient's uint16 setTimeout it doesn't silently truncate.
constexpr int HTTP_TIMEOUT_MS = 60000;
constexpr size_t READ_CHUNK = 2048;
constexpr size_t READ_CHUNK = 1024;
constexpr int MAX_REDIRECTS = 5;
struct Sink {
std::function<bool(const uint8_t*, size_t)> write; // returns false to abort the transfer
@@ -38,6 +45,68 @@ bool isRedirect(int status) {
return status == 301 || status == 302 || status == 303 || status == 307 || status == 308;
}
#if defined(FREEINK_NET_WOLFSSL)
HttpDownloader::DownloadError runGetWolf(const std::string& startUrl, const std::string& username,
const std::string& password, Sink& sink) {
std::string url = startUrl;
for (int hop = 0; hop <= MAX_REDIRECTS; ++hop) {
freeink::SecureHttpClient http;
http.setTimeout(HTTP_TIMEOUT_MS);
http.setInsecure();
if (!http.begin(url)) {
LOG_ERR("HTTP", "wolfSSL bad URL: %s", url.c_str());
return HttpDownloader::HTTP_ERROR;
}
http.addHeader("User-Agent", "CrossPoint-ESP32-" CROSSPOINT_VERSION);
if (!username.empty() && !password.empty()) {
const std::string credentials = username + ":" + password;
const String encoded = base64::encode(credentials.c_str());
http.addHeader("Authorization", std::string("Basic ") + encoded.c_str());
}
LOG_DBG("HTTP", "wolfSSL GET: %s", url.c_str());
const int status = http.GET(
[&http, &sink](const uint8_t* data, size_t len) {
if (http.getStatus() != 200) return true;
if (sink.total == 0 && http.hasContentLength()) sink.total = http.getContentLength();
if (!sink.write(data, len)) return false;
sink.downloaded += len;
if (sink.progress && sink.total > 0) sink.progress(sink.downloaded, sink.total);
return true;
},
[&sink]() { return sink.cancelFlag && *sink.cancelFlag; });
if (http.aborted()) return HttpDownloader::ABORTED;
if (status < 0) {
LOG_ERR("HTTP", "wolfSSL request failed: %s", url.c_str());
return HttpDownloader::HTTP_ERROR;
}
if (isRedirect(status)) {
const std::string location = http.getHeader("location");
if (location.empty() || !freeink::SecureHttpClient::resolveUrl(url, location, url)) {
LOG_ERR("HTTP", "wolfSSL bad redirect: %d", status);
return HttpDownloader::HTTP_ERROR;
}
continue;
}
if (status != 200) {
LOG_ERR("HTTP", "wolfSSL unexpected status: %d", status);
return HttpDownloader::HTTP_ERROR;
}
if (http.callbackAborted()) return HttpDownloader::FILE_ERROR;
if (!http.responseComplete()) {
LOG_ERR("HTTP", "wolfSSL incomplete: got %zu of %zu bytes", sink.downloaded, sink.total);
return HttpDownloader::HTTP_ERROR;
}
return HttpDownloader::OK;
}
LOG_ERR("HTTP", "too many redirects");
return HttpDownloader::HTTP_ERROR;
}
#endif
#if !defined(FREEINK_NET_WOLFSSL)
// Streams a GET body through sink.write in READ_CHUNK pieces. Uses the manual
// open/fetch_headers/read path rather than esp_http_client_perform(): perform()
// pushes the whole body through an event callback and reports a chunked body
@@ -84,8 +153,9 @@ HttpDownloader::DownloadError runGet(const std::string& url, const std::string&
}
int64_t contentLength = esp_http_client_fetch_headers(client);
int status = esp_http_client_get_status_code(client);
for (int hop = 0; isRedirect(status) && hop < 5; ++hop) {
for (int hop = 0; isRedirect(status) && hop < MAX_REDIRECTS; ++hop) {
if (esp_http_client_set_redirection(client) != ESP_OK) break;
esp_http_client_close(client);
err = esp_http_client_open(client, 0);
if (err != ESP_OK) {
LOG_ERR("HTTP", "redirect open failed: %s", esp_err_to_name(err));
@@ -141,6 +211,20 @@ HttpDownloader::DownloadError runGet(const std::string& url, const std::string&
}
return HttpDownloader::OK;
}
#endif // !FREEINK_NET_WOLFSSL
// All HTTP(S) fetches go through wolfSSL when it is the active TLS stack: it
// speaks TLS 1.3 and reads large bodies from servers where the esp_http_client/
// mbedTLS path fails to connect or stalls mid-stream. Plain-http URLs still use a
// WiFiClient inside runGetWolf, so this is safe for non-TLS targets too.
HttpDownloader::DownloadError runGetSecure(const std::string& url, const std::string& username,
const std::string& password, Sink& sink) {
#if defined(FREEINK_NET_WOLFSSL)
return runGetWolf(url, username, password, sink);
#else
return runGet(url, username, password, sink);
#endif
}
} // namespace
bool HttpDownloader::fetchUrl(const std::string& url, Stream& outContent, const std::string& username,
@@ -148,7 +232,7 @@ bool HttpDownloader::fetchUrl(const std::string& url, Stream& outContent, const
LOG_DBG("HTTP", "Fetching: %s", url.c_str());
Sink sink;
sink.write = [&outContent](const uint8_t* data, size_t len) { return outContent.write(data, len) == len; };
return runGet(url, username, password, sink) == OK;
return runGetSecure(url, username, password, sink) == OK;
}
bool HttpDownloader::fetchUrl(const std::string& url, std::string& outContent, const std::string& username,
@@ -160,7 +244,7 @@ bool HttpDownloader::fetchUrl(const std::string& url, std::string& outContent, c
outContent.append(reinterpret_cast<const char*>(data), len);
return true;
};
return runGet(url, username, password, sink) == OK;
return runGetSecure(url, username, password, sink) == OK;
}
bool HttpDownloader::fetchUrl(const std::string& url, const DataCallback& onData, const std::string& username,
@@ -168,7 +252,7 @@ bool HttpDownloader::fetchUrl(const std::string& url, const DataCallback& onData
LOG_DBG("HTTP", "Fetching: %s", url.c_str());
Sink sink;
sink.write = onData;
return runGet(url, username, password, sink) == OK;
return runGetSecure(url, username, password, sink) == OK;
}
HttpDownloader::DownloadError HttpDownloader::downloadToFile(const std::string& url, const std::string& destPath,
@@ -190,7 +274,7 @@ HttpDownloader::DownloadError HttpDownloader::downloadToFile(const std::string&
sink.cancelFlag = cancelFlag;
sink.write = [&file](const uint8_t* data, size_t len) { return file.write(data, len) == len; };
const DownloadError result = runGet(url, username, password, sink);
const DownloadError result = runGetSecure(url, username, password, sink);
// Close before any remove() on the same path; DESTRUCTOR_CLOSES_FILE would
// otherwise close only after the remove.
file.close();
+41 -53
View File
@@ -2,15 +2,12 @@
// clang-format off
// HttpDownloader.h pulls Arduino/SdFat, whose macros collide with lwip's
// ip4_addr.h unless seen before esp_http_client (which includes lwip). Pin this
// order; clang-format would otherwise sort the local header last and break the
// build.
// ip4_addr.h unless seen first. Pin this order; clang-format would otherwise sort
// the local header last and break the build.
#include "HttpDownloader.h"
#include <Logging.h>
#include <ReleaseJsonParser.h>
#include <esp_crt_bundle.h>
#include <esp_http_client.h>
#include <esp_https_ota.h>
#include <esp_ota_ops.h>
#include <esp_wifi.h>
// clang-format on
@@ -18,10 +15,6 @@
namespace {
constexpr char latestReleaseUrl[] = "https://api.github.com/repos/crosspoint-reader/crosspoint-reader/releases/latest";
esp_err_t http_client_set_header_cb(esp_http_client_handle_t http_client) {
return esp_http_client_set_header(http_client, "User-Agent", "CrossPoint-ESP32-" CROSSPOINT_VERSION);
}
} // namespace
OtaUpdater::OtaUpdaterError OtaUpdater::checkForUpdate() {
@@ -116,44 +109,39 @@ OtaUpdater::OtaUpdaterError OtaUpdater::installUpdate(ProgressCallback onProgres
return UPDATE_OLDER_ERROR;
}
esp_https_ota_handle_t ota_handle = NULL;
esp_err_t esp_err;
// esp_https_ota is hardwired to esp-tls/mbedTLS, whose precompiled build on this
// package can't negotiate TLS 1.3 (see SecureClient.h). Drive the OTA partition
// ourselves and stream the firmware through HttpDownloader, which runs over
// wolfSSL when FREEINK_NET_WOLFSSL is set, reusing its redirect handling for the
// GitHub -> CDN hop.
const esp_partition_t* updatePartition = esp_ota_get_next_update_partition(nullptr);
if (!updatePartition) {
LOG_ERR("OTA", "No OTA partition available");
return INTERNAL_UPDATE_ERROR;
}
esp_http_client_config_t client_config = {
.url = otaUrl.c_str(),
.timeout_ms = 15000,
// 4096 holds the github->CDN redirect headers (the 512 default truncates
// them); TX only carries our GET. Both are contiguous blocks contending
// with the TLS handshake on a tight internal arena, so keep them minimal.
.buffer_size = 4096,
.buffer_size_tx = 1024,
.skip_cert_common_name_check = true,
.crt_bundle_attach = esp_crt_bundle_attach,
.keep_alive_enable = true,
};
esp_https_ota_config_t ota_config = {
.http_config = &client_config,
.http_client_init_cb = http_client_set_header_cb,
};
esp_ota_handle_t otaHandle = 0;
esp_err_t esp_err = esp_ota_begin(updatePartition, OTA_SIZE_UNKNOWN, &otaHandle);
if (esp_err != ESP_OK) {
LOG_ERR("OTA", "esp_ota_begin failed: %s", esp_err_to_name(esp_err));
return INTERNAL_UPDATE_ERROR;
}
/* For better timing and connectivity, we disable power saving for WiFi */
esp_wifi_set_ps(WIFI_PS_NONE);
esp_err = esp_https_ota_begin(&ota_config, &ota_handle);
if (esp_err != ESP_OK) {
LOG_DBG("OTA", "HTTP OTA Begin Failed: %s", esp_err_to_name(esp_err));
return INTERNAL_UPDATE_ERROR;
}
processedSize = 0;
int lastReportedPct = -1;
do {
esp_err = esp_https_ota_perform(ota_handle);
processedSize = esp_https_ota_get_image_len_read(ota_handle);
// Fire the callback only on whole-percent change. Without this it fired
// every ~100ms perform iteration, waking the render task whose framebuffer
// work contends with TLS on the same internal arena. E-ink can't repaint
// faster than a percent tick anyway.
bool flashOk = true;
const bool fetchOk = HttpDownloader::fetchUrl(otaUrl, [&](const uint8_t* data, size_t len) {
if (esp_ota_write(otaHandle, data, len) != ESP_OK) {
flashOk = false;
return false; // abort the transfer
}
processedSize += len;
// Fire the callback only on whole-percent change. Per-chunk updates wake the
// render task, whose framebuffer work contends with TLS on the internal arena,
// and e-ink can't repaint faster than a percent tick anyway.
if (onProgress && totalSize > 0) {
const int pct = static_cast<int>(static_cast<uint64_t>(processedSize) * 100 / totalSize);
if (pct != lastReportedPct) {
@@ -161,27 +149,27 @@ OtaUpdater::OtaUpdaterError OtaUpdater::installUpdate(ProgressCallback onProgres
onProgress(ctx);
}
}
delay(100); // TODO: should we replace this with something better?
} while (esp_err == ESP_ERR_HTTPS_OTA_IN_PROGRESS);
return true;
});
/* Return back to default power saving for WiFi in case of failing */
esp_wifi_set_ps(WIFI_PS_MIN_MODEM);
if (esp_err != ESP_OK) {
LOG_ERR("OTA", "esp_https_ota_perform Failed: %s", esp_err_to_name(esp_err));
esp_https_ota_finish(ota_handle);
return HTTP_ERROR;
if (!fetchOk || !flashOk) {
LOG_ERR("OTA", "Firmware install failed (%s)", flashOk ? "download" : "flash write");
esp_ota_abort(otaHandle);
return flashOk ? HTTP_ERROR : INTERNAL_UPDATE_ERROR;
}
if (!esp_https_ota_is_complete_data_received(ota_handle)) {
LOG_ERR("OTA", "esp_https_ota_is_complete_data_received Failed: %s", esp_err_to_name(esp_err));
esp_https_ota_finish(ota_handle);
esp_err = esp_ota_end(otaHandle); // verifies the written image
if (esp_err != ESP_OK) {
LOG_ERR("OTA", "esp_ota_end failed: %s", esp_err_to_name(esp_err));
return INTERNAL_UPDATE_ERROR;
}
esp_err = esp_https_ota_finish(ota_handle);
esp_err = esp_ota_set_boot_partition(updatePartition);
if (esp_err != ESP_OK) {
LOG_ERR("OTA", "esp_https_ota_finish Failed: %s", esp_err_to_name(esp_err));
LOG_ERR("OTA", "esp_ota_set_boot_partition failed: %s", esp_err_to_name(esp_err));
return INTERNAL_UPDATE_ERROR;
}
+49 -5
View File
@@ -1,6 +1,29 @@
#include "UrlUtils.h"
#include <cstdio>
namespace UrlUtils {
namespace {
bool isHexDigit(const char c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); }
bool shouldEncode(const unsigned char c) {
if (c <= 0x20 || c >= 0x7f) return true;
switch (c) {
case '"':
case '<':
case '>':
case '\\':
case '^':
case '`':
case '{':
case '|':
case '}':
return true;
default:
return false;
}
}
} // namespace
std::string ensureProtocol(const std::string& url) {
if (url.find("://") == std::string::npos) {
@@ -22,18 +45,39 @@ std::string extractHost(const std::string& url) {
return pathStart == std::string::npos ? url : url.substr(0, pathStart);
}
std::string encodeUnsafeUrlChars(const std::string& url) {
std::string out;
out.reserve(url.size());
for (size_t i = 0; i < url.size(); ++i) {
const unsigned char c = static_cast<unsigned char>(url[i]);
if (c == '%' && i + 2 < url.size() && isHexDigit(url[i + 1]) && isHexDigit(url[i + 2])) {
out += url[i];
out += url[i + 1];
out += url[i + 2];
i += 2;
} else if (c == '%' || shouldEncode(c)) {
char encoded[4];
snprintf(encoded, sizeof(encoded), "%%%02X", c);
out += encoded;
} else {
out += static_cast<char>(c);
}
}
return out;
}
std::string buildUrl(const std::string& serverUrl, const std::string& path) {
// If path is already an absolute URL (has protocol), use it directly
if (path.find("://") != std::string::npos) {
return path;
return encodeUnsafeUrlChars(path);
}
const std::string urlWithProtocol = ensureProtocol(serverUrl);
if (path.empty()) {
return urlWithProtocol;
return encodeUnsafeUrlChars(urlWithProtocol);
}
if (path[0] == '/') {
// Absolute path - use just the host
return extractHost(urlWithProtocol) + path;
return encodeUnsafeUrlChars(extractHost(urlWithProtocol) + path);
}
// Relative path - strip query string from base before appending
std::string base = urlWithProtocol;
@@ -42,9 +86,9 @@ std::string buildUrl(const std::string& serverUrl, const std::string& path) {
base.resize(queryPos);
}
if (base.back() == '/') {
return base + path;
return encodeUnsafeUrlChars(base + path);
}
return base + "/" + path;
return encodeUnsafeUrlChars(base + "/" + path);
}
} // namespace UrlUtils
+5
View File
@@ -13,6 +13,11 @@ std::string ensureProtocol(const std::string& url);
*/
std::string extractHost(const std::string& url);
/**
* Percent-encode raw characters that esp_http_client rejects in a URL.
*/
std::string encodeUnsafeUrlChars(const std::string& url);
/**
* Build full URL from server URL and path.
* If path starts with /, it's an absolute path from the host root.