Merge pull request #200 from jpirnay/fix-heavydutymanga

refactor: css / AA memory usage
This commit is contained in:
jpirnay
2026-05-12 13:07:57 +02:00
committed by GitHub
13 changed files with 1376 additions and 332 deletions
+12
View File
@@ -259,6 +259,10 @@ int FontDecompressor::prewarmCache(const EpdFontData* fontData, const char* utf8
int32_t glyphIdx = findGlyphIndex(fontData, cp);
if (glyphIdx < 0) continue;
const EpdGlyph& glyph = fontData->glyph[glyphIdx];
// Whitespace/empty glyphs have no bitmap payload and do not need prewarm storage.
if (glyph.dataLength == 0 || glyph.width == 0 || glyph.height == 0) continue;
// Deduplicate against already prewarmed slots
bool alreadyCached = false;
for (uint8_t s = 0; s < pageSlotCount; s++) {
@@ -323,6 +327,8 @@ int FontDecompressor::prewarmCache(const EpdFontData* fontData, const char* utf8
int32_t outIdx = findGlyphIndex(fontData, fontData->ligaturePairs[li].ligatureCp);
if (outIdx < 0) continue;
const EpdGlyph& outGlyph = fontData->glyph[outIdx];
if (outGlyph.dataLength == 0 || outGlyph.width == 0 || outGlyph.height == 0) continue;
bool found = false;
for (uint16_t i = 0; i < glyphCount; i++) {
@@ -376,6 +382,12 @@ int FontDecompressor::prewarmCache(const EpdFontData* fontData, const char* utf8
stats.uniqueGroupsAccessed = groupCount;
// Safety: if the collected glyph set has no bitmap payload, skip slot allocation.
if (totalBytes == 0) {
LOG_DBG("FDC", "Prewarm skipped: %u glyphs but 0 bitmap bytes", glyphCount);
return 0;
}
// Sort neededGroups by ascending group index so flash reads are sequential.
// Uses insertion sort — groupCount is bounded at 128, typically <14 for Latin fonts.
for (uint8_t i = 1; i < groupCount; i++) {
+11 -5
View File
@@ -454,6 +454,11 @@ void Epub::parseCssFiles() const {
}
// No cache yet - parse CSS files
if (!cssParser->beginCacheCompile()) {
LOG_ERR("EBP", "Failed to start CSS compile pipeline");
return;
}
for (const auto& cssPath : cssFiles) {
LOG_DBG("EBP", "Parsing CSS file: %s", cssPath.c_str());
@@ -496,16 +501,17 @@ void Epub::parseCssFiles() const {
Storage.remove(tmpCssPath.c_str());
continue;
}
cssParser->loadFromStream(tempCssFile);
if (!cssParser->appendCompiledFromStream(tempCssFile)) {
LOG_ERR("EBP", "Failed to compile CSS file: %s", cssPath.c_str());
}
tempCssFile.close();
Storage.remove(tmpCssPath.c_str());
}
// Save to cache for next time
if (!cssParser->saveToCache()) {
LOG_ERR("EBP", "Failed to save CSS rules to cache");
// Finalize compact cache for next time.
if (!cssParser->endCacheCompile()) {
LOG_ERR("EBP", "Failed to finalize CSS rules cache");
}
cssParser->clear();
LOG_DBG("EBP", "Loaded %zu CSS style rules from %zu files", cssParser->ruleCount(), cssFiles.size());
}
+134 -22
View File
@@ -3,6 +3,8 @@
#include <HalStorage.h>
#include <Logging.h>
#include <Serialization.h>
#include <esp_heap_caps.h>
#include <esp_system.h>
#include <algorithm>
@@ -12,7 +14,7 @@
#include "parsers/ChapterHtmlSlimParser.h"
namespace {
constexpr uint8_t SECTION_FILE_VERSION = 26;
constexpr uint8_t SECTION_FILE_VERSION = 27;
constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(int) + // fontId
sizeof(float) + // lineCompression
@@ -20,6 +22,7 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(uint8_t) + // paragraphAlignment
sizeof(uint16_t) + // viewportWidth
sizeof(uint16_t) + // viewportHeight
sizeof(bool) + // parseComplete
sizeof(uint16_t) + // pageCount (stored as 16-bit in header)
sizeof(bool) + // hyphenationEnabled
sizeof(bool) + // embeddedStyle
@@ -29,6 +32,13 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + // SECTION_FILE_VERSION
sizeof(uint32_t) + // anchor map offset
sizeof(uint32_t); // paragraph LUT offset
constexpr uint32_t HEADER_TAIL_PARSE_COMPLETE_OFFSET =
HEADER_SIZE - sizeof(uint32_t) * 3 - sizeof(uint16_t) - sizeof(bool);
constexpr uint32_t HEADER_TAIL_PAGE_COUNT_OFFSET = HEADER_SIZE - sizeof(uint32_t) * 3 - sizeof(uint16_t);
constexpr uint32_t HEADER_TAIL_PAGE_LUT_OFFSET = HEADER_SIZE - sizeof(uint32_t) * 3;
constexpr uint32_t HEADER_TAIL_ANCHOR_OFFSET = HEADER_SIZE - sizeof(uint32_t) * 2;
constexpr uint32_t HEADER_TAIL_PARAGRAPH_LUT_OFFSET = HEADER_SIZE - sizeof(uint32_t);
// On-disk paragraph LUT entry: u32 xhtmlByteOffset + u16 paragraphIndex + u16 listItemIndex.
// listItemIndex is the running <li> count at page-break time; together with
// paragraphIndex it lets KOReader-supplied <p>- and <li>-anchored XPaths snap to
@@ -45,6 +55,20 @@ namespace {
constexpr uint32_t FNV_PRIME = 0x01000193; // 16777619
constexpr uint32_t FNV_OFFSET_BASIS = 0x811C9DC5; // 2166136261
// On constrained targets, loading the CSS rules map before chapter parsing can
// consume a large share of available heap and increase parse truncation risk.
// Allow compile-time override for tuning.
#ifndef SCT_EMBEDDED_STYLE_MIN_FREE_HEAP_BYTES
#define SCT_EMBEDDED_STYLE_MIN_FREE_HEAP_BYTES (96 * 1024)
#endif
#ifndef SCT_EMBEDDED_STYLE_MIN_CONTIG_HEAP_BYTES
#define SCT_EMBEDDED_STYLE_MIN_CONTIG_HEAP_BYTES (56 * 1024)
#endif
constexpr uint32_t EMBEDDED_STYLE_MIN_FREE_HEAP_BYTES = SCT_EMBEDDED_STYLE_MIN_FREE_HEAP_BYTES;
constexpr uint32_t EMBEDDED_STYLE_MIN_CONTIG_HEAP_BYTES = SCT_EMBEDDED_STYLE_MIN_CONTIG_HEAP_BYTES;
uint32_t fnv1a(const uint8_t* data, size_t length) {
uint32_t hash = FNV_OFFSET_BASIS;
for (size_t i = 0; i < length; ++i) {
@@ -195,9 +219,9 @@ void Section::writeSectionFileHeader(const int fontId, const float lineCompressi
}
static_assert(HEADER_SIZE == sizeof(SECTION_FILE_VERSION) + sizeof(fontId) + sizeof(lineCompression) +
sizeof(extraParagraphSpacing) + sizeof(paragraphAlignment) + sizeof(viewportWidth) +
sizeof(viewportHeight) + sizeof(pageCount) + sizeof(hyphenationEnabled) +
sizeof(embeddedStyle) + sizeof(bionicReadingEnabled) + sizeof(imageRendering) +
sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t),
sizeof(viewportHeight) + sizeof(bool) + sizeof(pageCount) +
sizeof(hyphenationEnabled) + sizeof(embeddedStyle) + sizeof(bionicReadingEnabled) +
sizeof(imageRendering) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t),
"Header size mismatch");
serialization::writePod(file, SECTION_FILE_VERSION);
serialization::writePod(file, fontId);
@@ -210,6 +234,7 @@ void Section::writeSectionFileHeader(const int fontId, const float lineCompressi
serialization::writePod(file, embeddedStyle);
serialization::writePod(file, bionicReadingEnabled);
serialization::writePod(file, imageRendering);
serialization::writePod(file, false); // Placeholder for parseComplete (patched later)
serialization::writePod(file, pageCount); // Placeholder for page count (will be initially 0, patched later)
serialization::writePod(file, static_cast<uint32_t>(0)); // Placeholder for LUT offset (patched later)
serialization::writePod(file, static_cast<uint32_t>(0)); // Placeholder for anchor map offset (patched later)
@@ -220,13 +245,30 @@ bool Section::loadSectionFile(const int fontId, const float lineCompression, con
const uint8_t paragraphAlignment, const uint16_t viewportWidth,
const uint16_t viewportHeight, const bool hyphenationEnabled, const bool embeddedStyle,
const bool bionicReadingEnabled, const uint8_t imageRendering) {
truncatedCache = false;
uint32_t propertyHash =
calculatePropertyHash(fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
viewportHeight, hyphenationEnabled, embeddedStyle, bionicReadingEnabled, imageRendering);
filePath = getSectionFilePath(propertyHash);
bool usingEmbeddedStyleFallback = false;
if (!Storage.openFileForRead("SCT", filePath, file)) {
return false;
// Fallback: allow loading a no-CSS cache variant when embedded CSS is enabled.
if (embeddedStyle) {
const uint32_t fallbackHash =
calculatePropertyHash(fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
viewportHeight, hyphenationEnabled, false, bionicReadingEnabled, imageRendering);
const std::string fallbackPath = getSectionFilePath(fallbackHash);
if (Storage.openFileForRead("SCT", fallbackPath, file)) {
filePath = fallbackPath;
usingEmbeddedStyleFallback = true;
LOG_INF("SCT", "Using no-CSS section cache fallback: %s", filePath.c_str());
} else {
return false;
}
} else {
return false;
}
}
// Match parameters
@@ -248,6 +290,7 @@ bool Section::loadSectionFile(const int fontId, const float lineCompression, con
bool fileEmbeddedStyle;
bool fileBionicReadingEnabled;
uint8_t fileImageRendering;
bool fileParseComplete;
serialization::readPod(file, fileFontId);
serialization::readPod(file, fileLineCompression);
serialization::readPod(file, fileExtraParagraphSpacing);
@@ -258,16 +301,21 @@ bool Section::loadSectionFile(const int fontId, const float lineCompression, con
serialization::readPod(file, fileEmbeddedStyle);
serialization::readPod(file, fileBionicReadingEnabled);
serialization::readPod(file, fileImageRendering);
serialization::readPod(file, fileParseComplete);
const bool embeddedStyleMatches =
(embeddedStyle == fileEmbeddedStyle) || (usingEmbeddedStyleFallback && !fileEmbeddedStyle);
if (fontId != fileFontId || lineCompression != fileLineCompression ||
extraParagraphSpacing != fileExtraParagraphSpacing || paragraphAlignment != fileParagraphAlignment ||
viewportWidth != fileViewportWidth || viewportHeight != fileViewportHeight ||
hyphenationEnabled != fileHyphenationEnabled || embeddedStyle != fileEmbeddedStyle ||
hyphenationEnabled != fileHyphenationEnabled || !embeddedStyleMatches ||
bionicReadingEnabled != fileBionicReadingEnabled || imageRendering != fileImageRendering) {
LOG_ERR("SCT", "Deserialization failed: Parameters do not match");
clearCache(); // closes file before removal
return false;
}
truncatedCache = !fileParseComplete;
}
serialization::readPod(file, pageCount);
@@ -309,8 +357,10 @@ bool Section::loadSectionFile(const int fontId, const float lineCompression, con
bool Section::clearCache() {
file.close(); // Must be closed before removal on FAT32
lut.clear();
tocBoundaries.clear();
pageCount = 0;
currentPage = 0;
truncatedCache = false;
if (!Storage.exists(filePath.c_str())) {
LOG_DBG("SCT", "Cache does not exist, no action needed");
@@ -330,7 +380,25 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
const uint8_t paragraphAlignment, const uint16_t viewportWidth,
const uint16_t viewportHeight, const bool hyphenationEnabled, const bool embeddedStyle,
const bool bionicReadingEnabled, const uint8_t imageRendering,
const std::function<void(int)>& progressFn) {
const std::function<void(int)>& progressFn, const bool skipEviction) {
if (!skipEviction) {
evictOldVariants();
}
if (embeddedStyle) {
const uint32_t freeHeap = esp_get_free_heap_size();
const uint32_t contigHeap = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
if (freeHeap < EMBEDDED_STYLE_MIN_FREE_HEAP_BYTES || contigHeap < EMBEDDED_STYLE_MIN_CONTIG_HEAP_BYTES) {
LOG_INF("SCT",
"Low heap for embedded CSS (free=%lu contig=%lu, need free>=%lu contig>=%lu); "
"building no-CSS section cache",
freeHeap, contigHeap, static_cast<uint32_t>(EMBEDDED_STYLE_MIN_FREE_HEAP_BYTES),
static_cast<uint32_t>(EMBEDDED_STYLE_MIN_CONTIG_HEAP_BYTES));
return createSectionFile(fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
viewportHeight, hyphenationEnabled, false, bionicReadingEnabled, imageRendering,
progressFn, true);
}
}
uint32_t propertyHash =
calculatePropertyHash(fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
viewportHeight, hyphenationEnabled, embeddedStyle, bionicReadingEnabled, imageRendering);
@@ -353,9 +421,6 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
return false;
}
// Evict old variants for this spine to keep cache size controlled BEFORE creating the new one
evictOldVariants();
if (!Storage.openFileForWrite("SCT", filePath, file)) {
return false;
}
@@ -375,6 +440,7 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
if (!cssParser->loadFromCache()) {
LOG_ERR("SCT", "Failed to load CSS from cache");
}
cssParser->resetResolveStats();
}
}
@@ -413,7 +479,13 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
const uint32_t phaseParseStart = millis();
const bool streamOk = epub->readItemContentsToStream(localPath, visitor, 1024);
const bool finalizeOk = visitor.finalize();
bool success = streamOk && finalizeOk && visitor.streamSucceeded();
const bool parserStreamOk = visitor.streamSucceeded();
if (cssParser) {
cssParser->logResolveStats(localPath.c_str());
}
const bool parseComplete = streamOk && finalizeOk && parserStreamOk;
bool success = parseComplete;
const bool hasParsedPages = pageCount > 0;
const uint32_t parseMs = millis() - phaseParseStart;
// streamMs is no longer a separate phase (SD-write of temp file is gone); keep the
// log breakdown stable by reporting it as 0.
@@ -421,13 +493,35 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
const uint32_t phaseFinalizeStart = millis();
if (!success) {
LOG_ERR("SCT", "Failed to parse XML and build pages (stream=%d finalize=%d)", streamOk ? 1 : 0, finalizeOk ? 1 : 0);
file.close();
Storage.remove(filePath.c_str());
if (cssParser) {
cssParser->clear();
// If parsing fails mid-stream due low memory but some pages were already serialized,
// keep the partial section cache so the chapter remains readable instead of failing hard.
if (hasParsedPages) {
LOG_ERR("SCT", "Parse incomplete; keeping partial section cache with %u pages (stream=%d finalize=%d parser=%d)",
pageCount, streamOk ? 1 : 0, finalizeOk ? 1 : 0, parserStreamOk ? 1 : 0);
success = true;
} else if (embeddedStyle) {
LOG_ERR("SCT",
"Parse failed with embedded CSS enabled; retrying section creation with embeddedStyle=0 "
"(stream=%d finalize=%d parser=%d)",
streamOk ? 1 : 0, finalizeOk ? 1 : 0, parserStreamOk ? 1 : 0);
file.close();
Storage.remove(filePath.c_str());
if (cssParser) {
cssParser->clear();
}
return createSectionFile(fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
viewportHeight, hyphenationEnabled, false, bionicReadingEnabled, imageRendering,
progressFn, true);
} else {
LOG_ERR("SCT", "Failed to parse XML and build pages (stream=%d finalize=%d parser=%d)", streamOk ? 1 : 0,
finalizeOk ? 1 : 0, parserStreamOk ? 1 : 0);
file.close();
Storage.remove(filePath.c_str());
if (cssParser) {
cssParser->clear();
}
return false;
}
return false;
}
const uint32_t fileSize = static_cast<uint32_t>(inflatedSize);
@@ -477,14 +571,31 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
serialization::writePod(file, entry.listItemIndex);
}
// Patch header with final pageCount, lutOffset, anchorMapOffset, and paragraphLutOffset
file.seek(HEADER_SIZE - sizeof(uint32_t) * 3 - sizeof(pageCount));
// Patch header with final parseComplete/pageCount and offsets.
const size_t headerPatchStart = HEADER_TAIL_PARSE_COMPLETE_OFFSET;
if (!file.seek(headerPatchStart)) {
LOG_ERR("SCT", "Failed to seek to section header patch offset %u", HEADER_TAIL_PARSE_COMPLETE_OFFSET);
file.close();
Storage.remove(filePath.c_str());
return false;
}
serialization::writePod(file, parseComplete);
serialization::writePod(file, pageCount);
serialization::writePod(file, lutOffset);
serialization::writePod(file, anchorMapOffset);
serialization::writePod(file, paragraphLutOffset);
file.flush();
const size_t expectedHeaderPatchEnd = headerPatchStart + sizeof(parseComplete) + sizeof(pageCount) +
sizeof(lutOffset) + sizeof(anchorMapOffset) + sizeof(paragraphLutOffset);
if (file.position() != expectedHeaderPatchEnd) {
LOG_ERR("SCT", "Section header patch write failed: wrote %u bytes at offset %u",
static_cast<unsigned>(file.position() - headerPatchStart), static_cast<unsigned>(headerPatchStart));
file.close();
Storage.remove(filePath.c_str());
return false;
}
if (cssParser) {
cssParser->clear();
}
@@ -499,6 +610,7 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
LOG_ERR("SCT", "Failed to open section file for reading after creation");
return false;
}
truncatedCache = !parseComplete;
this->lut = std::move(lut);
const uint32_t finalizeMs = millis() - phaseFinalizeStart;
const uint32_t totalMs = millis() - phaseTotalStart;
@@ -619,7 +731,7 @@ void Section::buildTocBoundariesFromFile(FsFile& f) {
// Single pass through on-disk anchors, matching against cached TOC anchors.
// Stop early once all TOC anchors are resolved.
// Header layout: ... | lutOffset (u32) | anchorMapOffset (u32) | paragraphLutOffset (u32) |
f.seek(HEADER_SIZE - sizeof(uint32_t) * 2);
f.seek(HEADER_TAIL_ANCHOR_OFFSET);
uint32_t anchorMapOffset;
serialization::readPod(f, anchorMapOffset);
@@ -689,7 +801,7 @@ std::optional<uint16_t> Section::getPageForAnchor(const std::string& anchor) con
}
const uint32_t fileSize = f.size();
f.seek(HEADER_SIZE - sizeof(uint32_t) * 2);
f.seek(HEADER_TAIL_ANCHOR_OFFSET);
uint32_t anchorMapOffset;
serialization::readPod(f, anchorMapOffset);
if (anchorMapOffset == 0 || anchorMapOffset >= fileSize) {
@@ -722,7 +834,7 @@ bool Section::readParagraphLutHeader(FsFile& outFile, uint16_t& outCount, uint32
const uint32_t fileSize = outFile.size();
outFile.seek(HEADER_SIZE - sizeof(uint32_t));
outFile.seek(HEADER_TAIL_PARAGRAPH_LUT_OFFSET);
uint32_t paragraphLutOffset;
serialization::readPod(outFile, paragraphLutOffset);
if (fileSize < sizeof(uint16_t) || paragraphLutOffset == 0 || paragraphLutOffset > fileSize - sizeof(uint16_t)) {
+3 -1
View File
@@ -17,6 +17,7 @@ class Section {
std::string filePath;
FsFile file;
std::vector<uint32_t> lut; // Cached page byte-offsets; loaded once, avoids per-page LUT seek
bool truncatedCache = false;
void writeSectionFileHeader(int fontId, float lineCompression, bool extraParagraphSpacing, uint8_t paragraphAlignment,
uint16_t viewportWidth, uint16_t viewportHeight, bool hyphenationEnabled,
@@ -66,8 +67,9 @@ class Section {
bool createSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, uint8_t paragraphAlignment,
uint16_t viewportWidth, uint16_t viewportHeight, bool hyphenationEnabled, bool embeddedStyle,
bool bionicReadingEnabled, uint8_t imageRendering,
const std::function<void(int)>& progressFn = nullptr);
const std::function<void(int)>& progressFn = nullptr, bool skipEviction = false);
std::unique_ptr<Page> loadPageFromSectionFile();
bool isTruncatedCache() const { return truncatedCache; }
// Given a page in this section, return the TOC index for that page.
int getTocIndexForPage(int page) const;
@@ -26,6 +26,7 @@ struct JpegContext {
const RenderConfig* config{nullptr};
int screenWidth{0};
int screenHeight{0};
ImageDitherMode effectiveDitherMode{ImageDitherMode::Bayer};
// Source dimensions after JPEGDEC's built-in scaling
int scaledSrcWidth{0};
@@ -103,7 +104,7 @@ uint8_t ditherGray(JpegContext& ctx, uint8_t gray, int localX, int outX, int out
return quantizeGray4Level(gray);
}
switch (ctx.config->ditherMode) {
switch (ctx.effectiveDitherMode) {
case ImageDitherMode::Atkinson:
if (ctx.atkinsonDitherer) {
return ctx.atkinsonDitherer->processPixel(gray, localX);
@@ -179,6 +180,70 @@ int32_t jpegSeek(JPEGFILE* pFile, int32_t pos) {
constexpr size_t JPEG_DECODER_APPROX_SIZE = 20 * 1024;
constexpr size_t MIN_FREE_HEAP_FOR_JPEG = JPEG_DECODER_APPROX_SIZE + 16 * 1024;
// Optional memory-behavior knobs for embedded targets.
#ifndef JPEG_ENABLE_FIRST_RENDER_NO_CACHE
#define JPEG_ENABLE_FIRST_RENDER_NO_CACHE 1
#endif
#ifndef JPEG_CACHE_MIN_FREE_HEAP_MARGIN
#define JPEG_CACHE_MIN_FREE_HEAP_MARGIN (24 * 1024)
#endif
#ifndef JPEG_CACHE_MIN_MAX_ALLOC_MARGIN
#define JPEG_CACHE_MIN_MAX_ALLOC_MARGIN (20 * 1024)
#endif
#ifndef JPEG_DITHER_LOW_MEM_MIN_FREE_HEAP
#define JPEG_DITHER_LOW_MEM_MIN_FREE_HEAP (MIN_FREE_HEAP_FOR_JPEG + 8 * 1024)
#endif
#ifndef JPEG_DITHER_LOW_MEM_MIN_MAX_ALLOC
#define JPEG_DITHER_LOW_MEM_MIN_MAX_ALLOC (48 * 1024)
#endif
size_t jpegCacheBytes(int width, int height) {
return static_cast<size_t>((width + 3) / 4) * static_cast<size_t>(height);
}
bool shouldEnableJpegCache(const RenderConfig& config, int width, int height) {
if (config.cachePath.empty()) return false;
#if JPEG_ENABLE_FIRST_RENDER_NO_CACHE
if (!Storage.exists(config.cachePath.c_str())) {
LOG_DBG("JPG", "No existing JPEG cache file on first render; enabling cache write: %s", config.cachePath.c_str());
}
#endif
const size_t cacheBytes = jpegCacheBytes(width, height);
const size_t freeHeap = ESP.getFreeHeap();
const size_t maxAlloc = ESP.getMaxAllocHeap();
const size_t minFreeForCaching = MIN_FREE_HEAP_FOR_JPEG + JPEG_CACHE_MIN_FREE_HEAP_MARGIN;
const size_t minMaxAllocForCaching = cacheBytes + JPEG_CACHE_MIN_MAX_ALLOC_MARGIN;
if (freeHeap < minFreeForCaching) {
LOG_DBG("JPG", "Skipping cache: free heap %u < %u (cache %u bytes)", static_cast<unsigned>(freeHeap),
static_cast<unsigned>(minFreeForCaching), static_cast<unsigned>(cacheBytes));
return false;
}
if (maxAlloc < minMaxAllocForCaching) {
LOG_DBG("JPG", "Skipping cache: max alloc %u < %u (cache %u bytes)", static_cast<unsigned>(maxAlloc),
static_cast<unsigned>(minMaxAllocForCaching), static_cast<unsigned>(cacheBytes));
return false;
}
return true;
}
bool shouldForceBayerDither(const RenderConfig& config) {
if (!config.useDithering) return false;
if (config.ditherMode == ImageDitherMode::Bayer) return false;
const size_t freeHeap = ESP.getFreeHeap();
const size_t maxAlloc = ESP.getMaxAllocHeap();
return freeHeap < JPEG_DITHER_LOW_MEM_MIN_FREE_HEAP || maxAlloc < JPEG_DITHER_LOW_MEM_MIN_MAX_ALLOC;
}
bool readJpegDimensionsFromHeader(const std::string& imagePath, ImageDimensions& out) {
FsFile f;
if (!Storage.openFileForRead("JPG", imagePath, f)) {
@@ -526,6 +591,7 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat
ctx.config = &config;
ctx.screenWidth = renderer.getScreenWidth();
ctx.screenHeight = renderer.getScreenHeight();
ctx.effectiveDitherMode = config.ditherMode;
int rc = jpeg->open(imagePath.c_str(), jpegOpen, jpegClose, jpegRead, jpegSeek, jpegDrawCallback);
if (rc != 1) {
@@ -605,7 +671,7 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat
jpeg->setUserPointer(&ctx);
// Allocate cache buffer using final output dimensions
ctx.caching = !config.cachePath.empty();
ctx.caching = shouldEnableJpegCache(config, destWidth, destHeight);
if (ctx.caching) {
if (!ctx.cache.allocate(destWidth, destHeight, config.x, config.y)) {
LOG_ERR("JPG", "Failed to allocate cache buffer, continuing without caching");
@@ -613,6 +679,12 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat
}
}
if (shouldForceBayerDither(config)) {
LOG_DBG("JPG", "Low-memory mode: forcing Bayer dithering (%u free, %u max alloc)",
static_cast<unsigned>(ESP.getFreeHeap()), static_cast<unsigned>(ESP.getMaxAllocHeap()));
ctx.effectiveDitherMode = ImageDitherMode::Bayer;
}
// See PngToFramebufferConverter for rationale: BW-only display needs a 1-bit
// dither so mid-grays don't collapse to black under DirectPixelWriter's `< 3` rule.
if (config.monochromeOutput) {
@@ -624,7 +696,7 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat
if (config.useDithering && !ctx.atkinson1BitDitherer) {
#ifdef ENABLE_IMAGE_DITHERING_EXTENSION
switch (config.ditherMode) {
switch (ctx.effectiveDitherMode) {
case ImageDitherMode::Atkinson:
ctx.atkinsonDitherer.reset(new (std::nothrow) AtkinsonDitherer(destWidth));
if (!ctx.atkinsonDitherer) {
+613 -243
View File
@@ -43,15 +43,94 @@ constexpr size_t MAX_RULES = 1500;
// Minimum free heap required to apply CSS during rendering
// If below this threshold, we skip CSS to avoid display artifacts.
constexpr size_t MIN_FREE_HEAP_FOR_CSS = 48 * 1024;
#ifndef CSS_MIN_FREE_HEAP_FOR_CSS
#define CSS_MIN_FREE_HEAP_FOR_CSS (40 * 1024)
#endif
constexpr size_t MIN_FREE_HEAP_FOR_CSS = CSS_MIN_FREE_HEAP_FOR_CSS;
// In-memory CSS rule cache sizing for disk-backed lookup mode.
// Keeps memory bounded on large books while retaining hot selectors.
#ifndef CSS_HOT_RULE_CACHE_SIZE
#define CSS_HOT_RULE_CACHE_SIZE 128
#endif
#ifndef CSS_NEGATIVE_CACHE_SIZE
#define CSS_NEGATIVE_CACHE_SIZE 256
#endif
constexpr size_t HOT_RULE_CACHE_SIZE = CSS_HOT_RULE_CACHE_SIZE;
constexpr size_t NEGATIVE_CACHE_SIZE = CSS_NEGATIVE_CACHE_SIZE;
// Maximum length for a single selector string
// Prevents parsing of extremely long or malformed selectors
constexpr size_t MAX_SELECTOR_LENGTH = 256;
constexpr size_t CSS_LENGTH_FIELD_COUNT = 11;
constexpr size_t CSS_LENGTH_BYTES = sizeof(float) + sizeof(uint8_t);
constexpr size_t CSS_FIXED_STYLE_BYTES =
4 * sizeof(uint8_t) + (CSS_LENGTH_FIELD_COUNT * CSS_LENGTH_BYTES) + sizeof(uint8_t) + sizeof(uint16_t);
static_assert(CSS_FIXED_STYLE_BYTES == 4 * sizeof(uint8_t) + (CSS_LENGTH_FIELD_COUNT * CSS_LENGTH_BYTES) +
sizeof(uint8_t) + sizeof(uint16_t),
"CSS_FIXED_STYLE_BYTES must match the compiled style payload layout");
// Cache file name (version is CssParser::CSS_CACHE_VERSION)
constexpr char rulesCache[] = "/css_rules.cache";
constexpr char compileTempRulesCache[] = "/css_rules.compile.tmp";
// Check if character is CSS whitespace
bool isCssWhitespace(const char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; }
// Resolver supports only: tag, .class, tag.class
bool isSelectorUsableByResolver(std::string_view selector) {
if (selector.empty()) {
return false;
}
if (selector.find_first_of("+>[:#~* ") != std::string_view::npos) {
return false;
}
const size_t dotPos = selector.find('.');
if (dotPos == std::string_view::npos) {
return true; // tag
}
if (dotPos == 0) {
return selector.size() > 1 && selector.find('.', 1) == std::string_view::npos; // .class only
}
// tag.class only, no additional dots
return dotPos + 1 < selector.size() && selector.find('.', dotPos + 1) == std::string_view::npos;
}
template <typename Fn>
void forEachNormalizedClassToken(const std::string& classAttr, std::string& normalizedBuf, Fn&& fn) {
size_t i = 0;
while (i < classAttr.size()) {
while (i < classAttr.size() && isCssWhitespace(classAttr[i])) {
++i;
}
if (i >= classAttr.size()) {
break;
}
const size_t start = i;
while (i < classAttr.size() && !isCssWhitespace(classAttr[i])) {
++i;
}
normalizedBuf.clear();
normalizedBuf.reserve(i - start);
for (size_t j = start; j < i; ++j) {
normalizedBuf.push_back(static_cast<char>(std::tolower(static_cast<unsigned char>(classAttr[j]))));
}
if (!normalizedBuf.empty()) {
fn(normalizedBuf);
}
}
}
std::string_view stripTrailingImportant(std::string_view value) {
constexpr std::string_view IMPORTANT = "!important";
@@ -384,9 +463,11 @@ void CssParser::processRuleBlockWithStyle(const std::string& selectorGroup, cons
const auto selectors = splitOnChar(selectorGroup, ',');
for (const auto& sel : selectors) {
totalSelectorCandidates_++;
// Validate selector length before processing
if (sel.size() > MAX_SELECTOR_LENGTH) {
LOG_DBG("CSS", "Selector too long (%zu > %zu), skipping", sel.size(), MAX_SELECTOR_LENGTH);
unsupportedSelectorSkips_++;
continue;
}
@@ -394,63 +475,50 @@ void CssParser::processRuleBlockWithStyle(const std::string& selectorGroup, cons
std::string key = normalized(sel);
if (key.empty()) continue;
// TODO: Consider adding support for sibling css selectors in the future
// Ensure no + in selector as we don't support adjacent CSS selectors for now
if (key.find('+') != std::string_view::npos) {
continue;
}
// TODO: Consider adding support for direct nested css selectors in the future
// Ensure no > in selector as we don't support nested CSS selectors for now
if (key.find('>') != std::string_view::npos) {
continue;
}
// TODO: Consider adding support for attribute css selectors in the future
// Ensure no [ in selector as we don't support attribute CSS selectors for now
if (key.find('[') != std::string_view::npos) {
continue;
}
// TODO: Consider adding support for pseudo selectors in the future
// Ensure no : in selector as we don't support pseudo CSS selectors for now
if (key.find(':') != std::string_view::npos) {
continue;
}
// TODO: Consider adding support for ID css selectors in the future
// Ensure no # in selector as we don't support ID CSS selectors for now
if (key.find('#') != std::string_view::npos) {
continue;
}
// TODO: Consider adding support for general sibling combinator selectors in the future
// Ensure no ~ in selector as we don't support general sibling combinator CSS selectors for now
if (key.find('~') != std::string_view::npos) {
continue;
}
// TODO: Consider adding support for wildcard css selectors in the future
// Ensure no * in selector as we don't support wildcard CSS selectors for now
if (key.find('*') != std::string_view::npos) {
continue;
}
// TODO: Add support for more complex selectors in the future
// At the moment, we only ever check for `tag`, `tag.class1` or `.class1`
// If the selector has whitespace in it, then it's either a CSS selector for a descendant element (e.g. `tag1 tag2`)
// or some other slightly more advanced CSS selector which we don't support yet
if (key.find(' ') != std::string_view::npos) {
if (!isSelectorUsableByResolver(key)) {
unsupportedSelectorSkips_++;
continue;
}
// Skip if this would exceed the rule limit
if (rulesBySelector_.size() >= MAX_RULES) {
const size_t ruleCount = compileModeActive_ ? compileSelectorOffsets_.size() : rulesBySelector_.size();
if (ruleCount >= MAX_RULES) {
LOG_DBG("CSS", "Reached max rules limit, stopping selector processing");
return;
}
// Store or merge with existing
if (compileModeActive_) {
if (!compileTempFile_) {
compileModeFailed_ = true;
continue;
}
compileTempFile_.flush();
CssStyle merged = style;
auto existingOffsetIt = compileSelectorOffsets_.find(key);
if (existingOffsetIt != compileSelectorOffsets_.end()) {
CssStyle existing;
FsFile tempRead;
if (Storage.openFileForRead("CSS", compileTempPath_, tempRead) && tempRead.seek(existingOffsetIt->second) &&
readCssStylePayload(tempRead, existing)) {
existing.applyOver(merged);
merged = existing;
} else {
LOG_ERR("CSS", "Failed to read compiled style for selector '%s' at offset %u", key.c_str(),
existingOffsetIt->second);
}
if (tempRead) {
tempRead.close();
}
}
const uint32_t styleOffset = compileTempFile_.position();
writeCssStylePayload(compileTempFile_, merged);
compileSelectorOffsets_[key] = styleOffset;
continue;
}
// Store or merge with existing (non-compile mode)
auto it = rulesBySelector_.find(key);
if (it != rulesBySelector_.end()) {
it->second.applyOver(style);
@@ -604,7 +672,443 @@ bool CssParser::loadFromStream(FsFile& source) {
handleChar('/');
}
LOG_DBG("CSS", "Parsed %zu rules from %zu bytes", rulesBySelector_.size(), totalRead);
if (compileModeActive_) {
LOG_DBG("CSS", "Parsed %zu usable selectors from %zu bytes (compile mode)", compileSelectorOffsets_.size(),
totalRead);
} else {
LOG_DBG("CSS", "Parsed %zu rules from %zu bytes", rulesBySelector_.size(), totalRead);
}
return true;
}
bool CssParser::beginCacheCompile() {
clear();
compileTempPath_ = cachePath + compileTempRulesCache;
Storage.remove(compileTempPath_.c_str());
if (!Storage.openFileForWrite("CSS", compileTempPath_, compileTempFile_)) {
return false;
}
compileSelectorOffsets_.clear();
compileModeActive_ = true;
compileModeFailed_ = false;
return true;
}
bool CssParser::appendCompiledFromStream(FsFile& source) {
if (!compileModeActive_) {
return false;
}
if (!loadFromStream(source)) {
compileModeFailed_ = true;
return false;
}
return !compileModeFailed_;
}
bool CssParser::endCacheCompile() {
if (!compileModeActive_) {
return false;
}
compileModeActive_ = false;
compileTempFile_.close();
if (compileModeFailed_) {
Storage.remove(compileTempPath_.c_str());
compileSelectorOffsets_.clear();
return false;
}
FsFile outFile;
if (!Storage.openFileForWrite("CSS", cachePath + rulesCache, outFile)) {
Storage.remove(compileTempPath_.c_str());
compileSelectorOffsets_.clear();
return false;
}
outFile.write(CssParser::CSS_CACHE_VERSION);
const auto ruleCount = static_cast<uint16_t>(compileSelectorOffsets_.size());
outFile.write(reinterpret_cast<const uint8_t*>(&ruleCount), sizeof(ruleCount));
outFile.write(reinterpret_cast<const uint8_t*>(&totalSelectorCandidates_), sizeof(totalSelectorCandidates_));
outFile.write(reinterpret_cast<const uint8_t*>(&unsupportedSelectorSkips_), sizeof(unsupportedSelectorSkips_));
FsFile tempFile;
if (!Storage.openFileForRead("CSS", compileTempPath_, tempFile)) {
outFile.close();
Storage.remove((cachePath + rulesCache).c_str());
Storage.remove(compileTempPath_.c_str());
compileSelectorOffsets_.clear();
return false;
}
std::array<uint8_t, CSS_FIXED_STYLE_BYTES> styleBytes{};
for (const auto& it : compileSelectorOffsets_) {
const auto selectorLen = static_cast<uint16_t>(it.first.size());
outFile.write(reinterpret_cast<const uint8_t*>(&selectorLen), sizeof(selectorLen));
outFile.write(reinterpret_cast<const uint8_t*>(it.first.data()), selectorLen);
if (!tempFile.seek(it.second)) {
tempFile.close();
outFile.close();
Storage.remove((cachePath + rulesCache).c_str());
Storage.remove(compileTempPath_.c_str());
compileSelectorOffsets_.clear();
return false;
}
if (tempFile.read(styleBytes.data(), styleBytes.size()) != static_cast<int>(styleBytes.size())) {
tempFile.close();
outFile.close();
Storage.remove((cachePath + rulesCache).c_str());
Storage.remove(compileTempPath_.c_str());
compileSelectorOffsets_.clear();
return false;
}
outFile.write(styleBytes.data(), styleBytes.size());
}
tempFile.close();
outFile.close();
Storage.remove(compileTempPath_.c_str());
compileSelectorOffsets_.clear();
rulesBySelector_.clear();
hotRuleCache_.clear();
hotRuleLru_.clear();
negativeRuleCache_.clear();
cacheRuleOffsets_.clear();
cacheIndexLoaded_ = false;
cachedRuleCount_ = 0;
return ensureCacheIndexLoaded();
}
bool CssParser::empty() const { return ruleCount() == 0; }
size_t CssParser::ruleCount() const {
if (!rulesBySelector_.empty()) {
return rulesBySelector_.size();
}
if (cacheIndexLoaded_) {
return cachedRuleCount_;
}
return 0;
}
void CssParser::clear() {
if (compileTempFile_) {
compileTempFile_.flush();
compileTempFile_.close();
}
if (!compileTempPath_.empty()) {
Storage.remove(compileTempPath_.c_str());
compileTempPath_.clear();
}
rulesBySelector_.clear();
cacheRuleOffsets_.clear();
hotRuleCache_.clear();
hotRuleLru_.clear();
negativeRuleCache_.clear();
cacheIndexLoaded_ = false;
cachedRuleCount_ = 0;
resolveStats_ = {};
compileModeActive_ = false;
compileModeFailed_ = false;
compileSelectorOffsets_.clear();
totalSelectorCandidates_ = 0;
unsupportedSelectorSkips_ = 0;
}
void CssParser::resetResolveStats() const { resolveStats_ = {}; }
CssParser::ResolveStats CssParser::getResolveStats() const { return resolveStats_; }
void CssParser::logResolveStats(const char* context) const {
const auto s = getResolveStats();
LOG_DBG("CSS",
"resolve stats[%s]: calls=%lu lowHeapSkips=%lu lowHeapRescuedHits=%lu lowHeapDiskBypasses=%lu "
"mapHits=%lu hotHits=%lu diskHits=%lu misses=%lu negativeHits=%lu "
"unsupportedSelectorsSkipped=%lu totalSelectorCandidates=%lu hotSize=%u indexSize=%u",
context ? context : "n/a", s.resolveCalls, s.lowHeapSkips, s.lowHeapRescuedHits, s.lowHeapDiskBypasses,
s.mapHits, s.hotHits, s.diskHits, s.misses, s.negativeHits,
static_cast<unsigned long>(unsupportedSelectorSkips_), static_cast<unsigned long>(totalSelectorCandidates_),
static_cast<unsigned>(hotRuleCache_.size()), static_cast<unsigned>(cachedRuleCount_));
}
bool CssParser::readCssStylePayload(FsFile& file, CssStyle& style) {
uint8_t enumVal;
if (file.read(&enumVal, 1) != 1) {
return false;
}
style.textAlign = static_cast<CssTextAlign>(enumVal);
if (file.read(&enumVal, 1) != 1) {
return false;
}
style.fontStyle = static_cast<CssFontStyle>(enumVal);
if (file.read(&enumVal, 1) != 1) {
return false;
}
style.fontWeight = static_cast<CssFontWeight>(enumVal);
if (file.read(&enumVal, 1) != 1) {
return false;
}
style.textDecoration = static_cast<CssTextDecoration>(enumVal);
auto readLength = [&file](CssLength& len) -> bool {
if (file.read(&len.value, sizeof(len.value)) != sizeof(len.value)) {
return false;
}
uint8_t unitVal;
if (file.read(&unitVal, 1) != 1) {
return false;
}
len.unit = static_cast<CssUnit>(unitVal);
return true;
};
if (!readLength(style.textIndent) || !readLength(style.marginTop) || !readLength(style.marginBottom) ||
!readLength(style.marginLeft) || !readLength(style.marginRight) || !readLength(style.paddingTop) ||
!readLength(style.paddingBottom) || !readLength(style.paddingLeft) || !readLength(style.paddingRight) ||
!readLength(style.imageHeight) || !readLength(style.imageWidth)) {
return false;
}
uint8_t displayVal;
if (file.read(&displayVal, 1) != 1) {
return false;
}
style.display = static_cast<CssDisplay>(displayVal);
uint16_t definedBits = 0;
if (file.read(&definedBits, sizeof(definedBits)) != sizeof(definedBits)) {
return false;
}
style.defined.textAlign = (definedBits & 1 << 0) != 0;
style.defined.fontStyle = (definedBits & 1 << 1) != 0;
style.defined.fontWeight = (definedBits & 1 << 2) != 0;
style.defined.textDecoration = (definedBits & 1 << 3) != 0;
style.defined.textIndent = (definedBits & 1 << 4) != 0;
style.defined.marginTop = (definedBits & 1 << 5) != 0;
style.defined.marginBottom = (definedBits & 1 << 6) != 0;
style.defined.marginLeft = (definedBits & 1 << 7) != 0;
style.defined.marginRight = (definedBits & 1 << 8) != 0;
style.defined.paddingTop = (definedBits & 1 << 9) != 0;
style.defined.paddingBottom = (definedBits & 1 << 10) != 0;
style.defined.paddingLeft = (definedBits & 1 << 11) != 0;
style.defined.paddingRight = (definedBits & 1 << 12) != 0;
style.defined.imageHeight = (definedBits & 1 << 13) != 0;
style.defined.imageWidth = (definedBits & 1 << 14) != 0;
style.defined.display = (definedBits & 1 << 15) != 0;
return true;
}
void CssParser::writeCssStylePayload(FsFile& file, const CssStyle& style) {
file.write(static_cast<uint8_t>(style.textAlign));
file.write(static_cast<uint8_t>(style.fontStyle));
file.write(static_cast<uint8_t>(style.fontWeight));
file.write(static_cast<uint8_t>(style.textDecoration));
auto writeLength = [&file](const CssLength& len) {
file.write(reinterpret_cast<const uint8_t*>(&len.value), sizeof(len.value));
file.write(static_cast<uint8_t>(len.unit));
};
writeLength(style.textIndent);
writeLength(style.marginTop);
writeLength(style.marginBottom);
writeLength(style.marginLeft);
writeLength(style.marginRight);
writeLength(style.paddingTop);
writeLength(style.paddingBottom);
writeLength(style.paddingLeft);
writeLength(style.paddingRight);
writeLength(style.imageHeight);
writeLength(style.imageWidth);
file.write(static_cast<uint8_t>(style.display));
uint16_t definedBits = 0;
if (style.defined.textAlign) definedBits |= 1 << 0;
if (style.defined.fontStyle) definedBits |= 1 << 1;
if (style.defined.fontWeight) definedBits |= 1 << 2;
if (style.defined.textDecoration) definedBits |= 1 << 3;
if (style.defined.textIndent) definedBits |= 1 << 4;
if (style.defined.marginTop) definedBits |= 1 << 5;
if (style.defined.marginBottom) definedBits |= 1 << 6;
if (style.defined.marginLeft) definedBits |= 1 << 7;
if (style.defined.marginRight) definedBits |= 1 << 8;
if (style.defined.paddingTop) definedBits |= 1 << 9;
if (style.defined.paddingBottom) definedBits |= 1 << 10;
if (style.defined.paddingLeft) definedBits |= 1 << 11;
if (style.defined.paddingRight) definedBits |= 1 << 12;
if (style.defined.imageHeight) definedBits |= 1 << 13;
if (style.defined.imageWidth) definedBits |= 1 << 14;
if (style.defined.display) definedBits |= 1 << 15;
file.write(reinterpret_cast<const uint8_t*>(&definedBits), sizeof(definedBits));
}
void CssParser::touchHotRule(const std::string& selector) const {
auto it = hotRuleCache_.find(selector);
if (it == hotRuleCache_.end()) {
return;
}
hotRuleLru_.erase(it->second.second);
hotRuleLru_.push_front(selector);
it->second.second = hotRuleLru_.begin();
}
void CssParser::cacheHotRule(const std::string& selector, const CssStyle& style) const {
auto it = hotRuleCache_.find(selector);
if (it != hotRuleCache_.end()) {
it->second.first = style;
touchHotRule(selector);
return;
}
hotRuleLru_.push_front(selector);
hotRuleCache_.emplace(selector, std::make_pair(style, hotRuleLru_.begin()));
if (hotRuleCache_.size() > HOT_RULE_CACHE_SIZE) {
const std::string& evictKey = hotRuleLru_.back();
hotRuleCache_.erase(evictKey);
hotRuleLru_.pop_back();
}
}
bool CssParser::readRuleFromDiskAtOffset(const uint32_t styleOffset, CssStyle& outStyle) const {
FsFile file;
if (!Storage.openFileForRead("CSS", cachePath + rulesCache, file)) {
return false;
}
if (!file.seek(styleOffset)) {
file.close();
return false;
}
const bool ok = readCssStylePayload(file, outStyle);
file.close();
return ok;
}
bool CssParser::lookupRule(const std::string& selector, CssStyle& outStyle, const bool allowDiskLookup) const {
auto mapIt = rulesBySelector_.find(selector);
if (mapIt != rulesBySelector_.end()) {
outStyle = mapIt->second;
resolveStats_.mapHits++;
return true;
}
auto hotIt = hotRuleCache_.find(selector);
if (hotIt != hotRuleCache_.end()) {
outStyle = hotIt->second.first;
touchHotRule(selector);
resolveStats_.hotHits++;
return true;
}
if (negativeRuleCache_.find(selector) != negativeRuleCache_.end()) {
resolveStats_.negativeHits++;
return false;
}
if (!allowDiskLookup) {
resolveStats_.lowHeapDiskBypasses++;
return false;
}
if (!ensureCacheIndexLoaded()) {
return false;
}
const auto offsetIt = cacheRuleOffsets_.find(selector);
if (offsetIt == cacheRuleOffsets_.end()) {
if (negativeRuleCache_.size() >= NEGATIVE_CACHE_SIZE) {
negativeRuleCache_.clear();
}
negativeRuleCache_.insert(selector);
return false;
}
if (!readRuleFromDiskAtOffset(offsetIt->second, outStyle)) {
return false;
}
cacheHotRule(selector, outStyle);
resolveStats_.diskHits++;
return true;
}
bool CssParser::ensureCacheIndexLoaded() const {
if (cacheIndexLoaded_) {
return true;
}
FsFile file;
if (!Storage.openFileForRead("CSS", cachePath + rulesCache, file)) {
return false;
}
uint8_t version = 0;
if (file.read(&version, 1) != 1 || version != CssParser::CSS_CACHE_VERSION) {
file.close();
Storage.remove((cachePath + rulesCache).c_str());
return false;
}
uint16_t ruleCount = 0;
if (file.read(&ruleCount, sizeof(ruleCount)) != sizeof(ruleCount) || ruleCount > MAX_RULES) {
file.close();
return false;
}
uint32_t totalCandidates = 0;
uint32_t unsupportedSkips = 0;
if (file.read(reinterpret_cast<uint8_t*>(&totalCandidates), sizeof(totalCandidates)) != sizeof(totalCandidates) ||
file.read(reinterpret_cast<uint8_t*>(&unsupportedSkips), sizeof(unsupportedSkips)) != sizeof(unsupportedSkips)) {
file.close();
return false;
}
cacheRuleOffsets_.clear();
cacheRuleOffsets_.reserve(ruleCount);
hotRuleCache_.clear();
hotRuleLru_.clear();
negativeRuleCache_.clear();
for (uint16_t i = 0; i < ruleCount; ++i) {
uint16_t selectorLen = 0;
if (file.read(&selectorLen, sizeof(selectorLen)) != sizeof(selectorLen) || selectorLen == 0 ||
selectorLen > MAX_SELECTOR_LENGTH) {
file.close();
cacheRuleOffsets_.clear();
return false;
}
std::string selector;
selector.resize(selectorLen);
if (file.read(&selector[0], selectorLen) != selectorLen) {
file.close();
cacheRuleOffsets_.clear();
return false;
}
const uint32_t styleOffset = file.position();
cacheRuleOffsets_[std::move(selector)] = styleOffset;
if (!file.seek(styleOffset + CSS_FIXED_STYLE_BYTES)) {
file.close();
cacheRuleOffsets_.clear();
return false;
}
}
cachedRuleCount_ = cacheRuleOffsets_.size();
totalSelectorCandidates_ = totalCandidates;
unsupportedSelectorSkips_ = unsupportedSkips;
cacheIndexLoaded_ = true;
file.close();
LOG_DBG("CSS", "Loaded CSS index: %u selectors (hot cache size=%u, unsupported=%lu/%lu)",
static_cast<unsigned>(cachedRuleCount_), static_cast<unsigned>(HOT_RULE_CACHE_SIZE),
static_cast<unsigned long>(unsupportedSelectorSkips_), static_cast<unsigned long>(totalSelectorCandidates_));
return true;
}
@@ -612,47 +1116,70 @@ bool CssParser::loadFromStream(FsFile& source) {
CssStyle CssParser::resolveStyle(const std::string& tagName, const std::string& classAttr) const {
static bool lowHeapWarningLogged = false;
if (ESP.getFreeHeap() < MIN_FREE_HEAP_FOR_CSS) {
resolveStats_.resolveCalls++;
const uint32_t freeHeap = ESP.getFreeHeap();
const bool lowHeapMode = freeHeap < MIN_FREE_HEAP_FOR_CSS;
if (lowHeapMode) {
if (!lowHeapWarningLogged) {
lowHeapWarningLogged = true;
LOG_DBG("CSS", "Warning: low heap (%u bytes) below MIN_FREE_HEAP_FOR_CSS (%u), returning empty style",
ESP.getFreeHeap(), static_cast<unsigned>(MIN_FREE_HEAP_FOR_CSS));
LOG_DBG("CSS", "Warning: low heap (%u bytes) below MIN_FREE_HEAP_FOR_CSS (%u), skipping disk CSS lookups",
freeHeap, static_cast<unsigned>(MIN_FREE_HEAP_FOR_CSS));
}
return CssStyle{};
resolveStats_.lowHeapSkips++;
}
CssStyle result;
const std::string tag = normalized(tagName);
// 1. Apply element-level style (lowest priority)
const auto tagIt = rulesBySelector_.find(tag);
if (tagIt != rulesBySelector_.end()) {
result.applyOver(tagIt->second);
{
CssStyle tagStyle;
if (lookupRule(tag, tagStyle, !lowHeapMode)) {
if (lowHeapMode) {
resolveStats_.lowHeapRescuedHits++;
}
result.applyOver(tagStyle);
}
}
// TODO: Support combinations of classes (e.g. style on .class1.class2)
// 2. Apply class styles (medium priority)
if (!classAttr.empty()) {
const auto classes = splitWhitespace(classAttr);
std::string classToken;
std::string classKey;
classKey.reserve(32);
std::string combinedKey;
combinedKey.reserve(tag.size() + 1 + 32);
for (const auto& cls : classes) {
std::string classKey = "." + normalized(cls);
forEachNormalizedClassToken(classAttr, classToken, [&](const std::string& cls) {
classKey.clear();
classKey.push_back('.');
classKey.append(cls);
auto classIt = rulesBySelector_.find(classKey);
if (classIt != rulesBySelector_.end()) {
result.applyOver(classIt->second);
CssStyle classStyle;
if (lookupRule(classKey, classStyle, !lowHeapMode)) {
if (lowHeapMode) {
resolveStats_.lowHeapRescuedHits++;
}
result.applyOver(classStyle);
}
}
// TODO: Support combinations of classes (e.g. style on p.class1.class2)
// 3. Apply element.class styles (higher priority)
for (const auto& cls : classes) {
std::string combinedKey = tag + "." + normalized(cls);
combinedKey.clear();
combinedKey.append(tag);
combinedKey.push_back('.');
combinedKey.append(cls);
auto combinedIt = rulesBySelector_.find(combinedKey);
if (combinedIt != rulesBySelector_.end()) {
result.applyOver(combinedIt->second);
CssStyle combinedStyle;
if (lookupRule(combinedKey, combinedStyle, !lowHeapMode)) {
if (lowHeapMode) {
resolveStats_.lowHeapRescuedHits++;
}
result.applyOver(combinedStyle);
}
}
});
}
if (!result.defined.anySet()) {
resolveStats_.misses++;
}
return result;
@@ -664,9 +1191,6 @@ CssStyle CssParser::parseInlineStyle(const std::string& styleValue) { return par
// Cache serialization
// Cache file name (version is CssParser::CSS_CACHE_VERSION)
constexpr char rulesCache[] = "/css_rules.cache";
bool CssParser::hasCache() const { return Storage.exists((cachePath + rulesCache).c_str()); }
void CssParser::deleteCache() const {
@@ -689,6 +1213,8 @@ bool CssParser::saveToCache() const {
// Write rule count
const auto ruleCount = static_cast<uint16_t>(rulesBySelector_.size());
file.write(reinterpret_cast<const uint8_t*>(&ruleCount), sizeof(ruleCount));
file.write(reinterpret_cast<const uint8_t*>(&totalSelectorCandidates_), sizeof(totalSelectorCandidates_));
file.write(reinterpret_cast<const uint8_t*>(&unsupportedSelectorSkips_), sizeof(unsupportedSelectorSkips_));
// Write each rule: selector string + CssStyle fields
for (const auto& pair : rulesBySelector_) {
@@ -754,175 +1280,19 @@ bool CssParser::loadFromCache() {
return false;
}
FsFile file;
if (!Storage.openFileForRead("CSS", cachePath + rulesCache, file)) {
// Drop parse-time in-memory rules, then initialize on-disk selector index.
rulesBySelector_.clear();
hotRuleCache_.clear();
hotRuleLru_.clear();
negativeRuleCache_.clear();
cacheRuleOffsets_.clear();
cacheIndexLoaded_ = false;
cachedRuleCount_ = 0;
if (!ensureCacheIndexLoaded()) {
return false;
}
// Clear existing rules
clear();
// Read and verify version
uint8_t version = 0;
if (file.read(&version, 1) != 1 || version != CssParser::CSS_CACHE_VERSION) {
LOG_DBG("CSS", "Cache version mismatch (got %u, expected %u), removing stale cache for rebuild", version,
CssParser::CSS_CACHE_VERSION);
file.close();
Storage.remove((cachePath + rulesCache).c_str());
return false;
}
// Read rule count
uint16_t ruleCount = 0;
if (file.read(&ruleCount, sizeof(ruleCount)) != sizeof(ruleCount)) {
file.close();
return false;
}
if (ruleCount > MAX_RULES) {
LOG_DBG("CSS", "Invalid cache rule count (%u > %zu)", ruleCount, MAX_RULES);
rulesBySelector_.clear();
file.close();
return false;
}
auto hasRemainingBytes = [&file](const size_t neededBytes) -> bool {
return static_cast<size_t>(file.available()) >= neededBytes;
};
constexpr size_t CSS_LENGTH_FIELD_COUNT = 11;
constexpr size_t CSS_LENGTH_BYTES = sizeof(float) + sizeof(uint8_t);
constexpr size_t CSS_FIXED_STYLE_BYTES =
4 * sizeof(uint8_t) + (CSS_LENGTH_FIELD_COUNT * CSS_LENGTH_BYTES) + sizeof(uint8_t) + sizeof(uint16_t);
// Read each rule
for (uint16_t i = 0; i < ruleCount; ++i) {
// Read selector string
uint16_t selectorLen = 0;
if (!hasRemainingBytes(sizeof(selectorLen))) {
rulesBySelector_.clear();
file.close();
return false;
}
if (file.read(&selectorLen, sizeof(selectorLen)) != sizeof(selectorLen)) {
rulesBySelector_.clear();
file.close();
return false;
}
if (selectorLen == 0 || selectorLen > MAX_SELECTOR_LENGTH || !hasRemainingBytes(selectorLen)) {
LOG_DBG("CSS", "Invalid selector length in cache: %u", selectorLen);
rulesBySelector_.clear();
file.close();
return false;
}
std::string selector;
selector.resize(selectorLen);
if (file.read(&selector[0], selectorLen) != selectorLen) {
rulesBySelector_.clear();
file.close();
return false;
}
if (!hasRemainingBytes(CSS_FIXED_STYLE_BYTES)) {
LOG_DBG("CSS", "Truncated CSS cache while reading style payload");
rulesBySelector_.clear();
file.close();
return false;
}
// Read CssStyle fields
CssStyle style;
uint8_t enumVal;
if (file.read(&enumVal, 1) != 1) {
rulesBySelector_.clear();
file.close();
return false;
}
style.textAlign = static_cast<CssTextAlign>(enumVal);
if (file.read(&enumVal, 1) != 1) {
rulesBySelector_.clear();
file.close();
return false;
}
style.fontStyle = static_cast<CssFontStyle>(enumVal);
if (file.read(&enumVal, 1) != 1) {
rulesBySelector_.clear();
file.close();
return false;
}
style.fontWeight = static_cast<CssFontWeight>(enumVal);
if (file.read(&enumVal, 1) != 1) {
rulesBySelector_.clear();
file.close();
return false;
}
style.textDecoration = static_cast<CssTextDecoration>(enumVal);
// Read CssLength fields
auto readLength = [&file](CssLength& len) -> bool {
if (file.read(&len.value, sizeof(len.value)) != sizeof(len.value)) {
return false;
}
uint8_t unitVal;
if (file.read(&unitVal, 1) != 1) {
return false;
}
len.unit = static_cast<CssUnit>(unitVal);
return true;
};
if (!readLength(style.textIndent) || !readLength(style.marginTop) || !readLength(style.marginBottom) ||
!readLength(style.marginLeft) || !readLength(style.marginRight) || !readLength(style.paddingTop) ||
!readLength(style.paddingBottom) || !readLength(style.paddingLeft) || !readLength(style.paddingRight) ||
!readLength(style.imageHeight) || !readLength(style.imageWidth)) {
rulesBySelector_.clear();
file.close();
return false;
}
// Read display value
uint8_t displayVal;
if (file.read(&displayVal, 1) != 1) {
rulesBySelector_.clear();
file.close();
return false;
}
style.display = static_cast<CssDisplay>(displayVal);
// Read defined flags
uint16_t definedBits = 0;
if (file.read(&definedBits, sizeof(definedBits)) != sizeof(definedBits)) {
rulesBySelector_.clear();
file.close();
return false;
}
style.defined.textAlign = (definedBits & 1 << 0) != 0;
style.defined.fontStyle = (definedBits & 1 << 1) != 0;
style.defined.fontWeight = (definedBits & 1 << 2) != 0;
style.defined.textDecoration = (definedBits & 1 << 3) != 0;
style.defined.textIndent = (definedBits & 1 << 4) != 0;
style.defined.marginTop = (definedBits & 1 << 5) != 0;
style.defined.marginBottom = (definedBits & 1 << 6) != 0;
style.defined.marginLeft = (definedBits & 1 << 7) != 0;
style.defined.marginRight = (definedBits & 1 << 8) != 0;
style.defined.paddingTop = (definedBits & 1 << 9) != 0;
style.defined.paddingBottom = (definedBits & 1 << 10) != 0;
style.defined.paddingLeft = (definedBits & 1 << 11) != 0;
style.defined.paddingRight = (definedBits & 1 << 12) != 0;
style.defined.imageHeight = (definedBits & 1 << 13) != 0;
style.defined.imageWidth = (definedBits & 1 << 14) != 0;
style.defined.display = (definedBits & 1 << 15) != 0;
rulesBySelector_[selector] = style;
}
LOG_DBG("CSS", "Loaded %u rules from cache", ruleCount);
file.close();
LOG_DBG("CSS", "Loaded %u rules from cache index", static_cast<unsigned>(cachedRuleCount_));
return true;
}
+60 -4
View File
@@ -2,8 +2,10 @@
#include <HalStorage.h>
#include <list>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
@@ -30,8 +32,20 @@
*/
class CssParser {
public:
struct ResolveStats {
uint32_t resolveCalls = 0;
uint32_t lowHeapSkips = 0;
uint32_t lowHeapRescuedHits = 0;
uint32_t lowHeapDiskBypasses = 0;
uint32_t mapHits = 0;
uint32_t hotHits = 0;
uint32_t diskHits = 0;
uint32_t misses = 0;
uint32_t negativeHits = 0;
};
// Bump when CSS cache format or rules change; section caches are invalidated when this changes
static constexpr uint8_t CSS_CACHE_VERSION = 4;
static constexpr uint8_t CSS_CACHE_VERSION = 5;
explicit CssParser(std::string cachePath) : cachePath(std::move(cachePath)) {}
~CssParser() = default;
@@ -68,17 +82,17 @@ class CssParser {
/**
* Check if any rules have been loaded
*/
[[nodiscard]] bool empty() const { return rulesBySelector_.empty(); }
[[nodiscard]] bool empty() const;
/**
* Get count of loaded rule sets
*/
[[nodiscard]] size_t ruleCount() const { return rulesBySelector_.size(); }
[[nodiscard]] size_t ruleCount() const;
/**
* Clear all loaded rules
*/
void clear() { rulesBySelector_.clear(); }
void clear();
/**
* Check if CSS rules cache file exists
@@ -103,12 +117,45 @@ class CssParser {
*/
bool loadFromCache();
// Low-memory CSS compilation pipeline:
// - beginCacheCompile(): starts streaming compile mode
// - appendCompiledFromStream(): parses one stylesheet stream into compile staging
// - endCacheCompile(): finalizes cache file from staged records
bool beginCacheCompile();
bool appendCompiledFromStream(FsFile& source);
bool endCacheCompile();
// CSS lookup telemetry helpers for tuning memory/caching behavior on-device.
void resetResolveStats() const;
[[nodiscard]] ResolveStats getResolveStats() const;
void logResolveStats(const char* context) const;
private:
// Storage: maps normalized selector -> style properties
std::unordered_map<std::string, CssStyle> rulesBySelector_;
std::string cachePath;
// Disk-backed CSS dictionary index: selector -> byte offset for serialized CssStyle payload.
// Built from cache file once, then styles are loaded on demand into hotRuleCache_.
mutable bool cacheIndexLoaded_ = false;
mutable size_t cachedRuleCount_ = 0;
mutable std::unordered_map<std::string, uint32_t> cacheRuleOffsets_;
mutable uint32_t totalSelectorCandidates_ = 0;
mutable uint32_t unsupportedSelectorSkips_ = 0;
// Bounded hot cache of most recently used rules.
mutable std::list<std::string> hotRuleLru_;
mutable std::unordered_map<std::string, std::pair<CssStyle, std::list<std::string>::iterator>> hotRuleCache_;
mutable std::unordered_set<std::string> negativeRuleCache_;
mutable ResolveStats resolveStats_;
bool compileModeActive_ = false;
bool compileModeFailed_ = false;
std::string compileTempPath_;
FsFile compileTempFile_;
std::unordered_map<std::string, uint32_t> compileSelectorOffsets_;
// Internal parsing helpers
void processRuleBlockWithStyle(const std::string& selectorGroup, const CssStyle& style);
static CssStyle parseDeclarations(const std::string& declBlock);
@@ -129,4 +176,13 @@ class CssParser {
static void normalizedInto(const std::string& s, std::string& out);
static std::vector<std::string> splitOnChar(const std::string& s, char delimiter);
static std::vector<std::string> splitWhitespace(const std::string& s);
// On-demand rule loading helpers
bool ensureCacheIndexLoaded() const;
bool lookupRule(const std::string& selector, CssStyle& outStyle, bool allowDiskLookup = true) const;
bool readRuleFromDiskAtOffset(uint32_t styleOffset, CssStyle& outStyle) const;
static bool readCssStylePayload(FsFile& file, CssStyle& style);
static void writeCssStylePayload(FsFile& file, const CssStyle& style);
void touchHotRule(const std::string& selector) const;
void cacheHotRule(const std::string& selector, const CssStyle& style) const;
};
+131 -29
View File
@@ -6,6 +6,7 @@
#include <HalStorage.h>
#include <Logging.h>
#include <Utf8.h>
#include <esp_heap_caps.h>
#include <expat.h>
#include <algorithm>
@@ -30,11 +31,35 @@ constexpr int NUM_HEADER_TAGS = sizeof(HEADER_TAGS) / sizeof(HEADER_TAGS[0]);
constexpr size_t MIN_SIZE_FOR_POPUP = 15 * 1024;
constexpr size_t SIZE_FOR_PROGRESS_HEARTBEAT = 30 * 1024;
constexpr size_t SIZE_FOR_PROGRESS_FINE = 80 * 1024;
constexpr size_t MIN_FREE_HEAP_FOR_INDEXING_POPUP = 32 * 1024;
constexpr size_t MIN_CONTIG_HEAP_FOR_INDEXING_POPUP = 12 * 1024;
constexpr size_t PARSE_BUFFER_SIZE = 1024;
constexpr size_t IMAGE_EXTRACT_CHUNK_SIZE = 1024;
constexpr size_t MIN_FREE_HEAP_FOR_IMAGE_EXTRACT = 48 * 1024;
constexpr size_t MIN_MAX_ALLOC_FOR_IMAGE_EXTRACT = 36 * 1024;
#ifndef EHP_TEXT_LAYOUT_SOFT_MIN_FREE_HEAP
#define EHP_TEXT_LAYOUT_SOFT_MIN_FREE_HEAP (18 * 1024)
#endif
#ifndef EHP_TEXT_LAYOUT_SOFT_MIN_MAX_ALLOC
#define EHP_TEXT_LAYOUT_SOFT_MIN_MAX_ALLOC (12 * 1024)
#endif
#ifndef EHP_TEXT_LAYOUT_HARD_MIN_FREE_HEAP
#define EHP_TEXT_LAYOUT_HARD_MIN_FREE_HEAP (9 * 1024)
#endif
#ifndef EHP_TEXT_LAYOUT_HARD_MIN_MAX_ALLOC
#define EHP_TEXT_LAYOUT_HARD_MIN_MAX_ALLOC (6 * 1024)
#endif
constexpr size_t MIN_FREE_HEAP_FOR_TEXT_LAYOUT = EHP_TEXT_LAYOUT_SOFT_MIN_FREE_HEAP;
constexpr size_t MIN_MAX_ALLOC_FOR_TEXT_LAYOUT = EHP_TEXT_LAYOUT_SOFT_MIN_MAX_ALLOC;
constexpr size_t MIN_FREE_HEAP_FOR_TEXT_LAYOUT_HARD = EHP_TEXT_LAYOUT_HARD_MIN_FREE_HEAP;
constexpr size_t MIN_MAX_ALLOC_FOR_TEXT_LAYOUT_HARD = EHP_TEXT_LAYOUT_HARD_MIN_MAX_ALLOC;
const char* BLOCK_TAGS[] = {"p", "li", "div", "br", "blockquote", "pre"};
constexpr int NUM_BLOCK_TAGS = sizeof(BLOCK_TAGS) / sizeof(BLOCK_TAGS[0]);
@@ -198,8 +223,43 @@ void ChapterHtmlSlimParser::updateEffectiveInlineStyle() {
}
}
bool ChapterHtmlSlimParser::ensureHeapForTextLayout(const char* phase) {
if (streamFailed) {
return false;
}
const uint32_t freeHeap = ESP.getFreeHeap();
const uint32_t maxAllocHeap = ESP.getMaxAllocHeap();
if (freeHeap >= MIN_FREE_HEAP_FOR_TEXT_LAYOUT && maxAllocHeap >= MIN_MAX_ALLOC_FOR_TEXT_LAYOUT) {
return true;
}
// Soft low-memory zone: keep parsing in degraded mode and only hard-abort when
// both free and contiguous heap fall to critical levels.
if (freeHeap >= MIN_FREE_HEAP_FOR_TEXT_LAYOUT_HARD && maxAllocHeap >= MIN_MAX_ALLOC_FOR_TEXT_LAYOUT_HARD) {
lowMemoryImageFallback = true;
LOG_DBG("EHP", "Low heap (%u free, %u max alloc) before %s; continuing in degraded mode", freeHeap, maxAllocHeap,
phase);
return true;
}
LOG_ERR("EHP", "Low heap (%u free, %u max alloc), aborting parse before %s", freeHeap, maxAllocHeap, phase);
streamFailed = true;
layoutFailed = true;
if (activeParser) {
XML_StopParser(activeParser, XML_FALSE);
}
return false;
}
// flush the contents of partWordBuffer to currentTextBlock
void ChapterHtmlSlimParser::flushPartWordBuffer() {
bool ChapterHtmlSlimParser::flushPartWordBuffer() {
if (streamFailed) {
partWordBufferIndex = 0;
nextWordContinues = false;
return false;
}
// Determine font style from depth-based tracking and CSS effective style
const bool isBold = boldUntilDepth < depth || effectiveBold;
const bool isItalic = italicUntilDepth < depth || effectiveItalic;
@@ -229,6 +289,11 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() {
currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextWordContinues);
if (currentTextBlock->size() > 96) {
if (!ensureHeapForTextLayout("long-block split")) {
partWordBufferIndex = 0;
nextWordContinues = false;
return;
}
LOG_DBG("EHP", "Text block too long, splitting into multiple pages");
const int horizontalInset = currentTextBlock->getBlockStyle().totalHorizontalInset();
const uint16_t effectiveWidth =
@@ -244,6 +309,7 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() {
}
partWordBufferIndex = 0;
nextWordContinues = false;
return true;
}
// Emit the current page, keeping paragraphLutPerPage and completedPageCount in lockstep.
@@ -317,6 +383,10 @@ void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) {
void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* name, const XML_Char** atts) {
auto* self = static_cast<ChapterHtmlSlimParser*>(userData);
if (self->streamFailed) {
return;
}
// Middle of skip
if (self->skipUntilDepth < self->depth) {
self->depth += 1;
@@ -389,7 +459,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
}
// Flush any pending text before starting the table
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
if (self->currentTextBlock && !self->currentTextBlock->isEmpty()) {
self->makePages();
@@ -411,7 +481,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
if (self->currentTable && self->currentTable->depth == 1 && (strcmp(name, "td") == 0 || strcmp(name, "th") == 0)) {
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
if (self->currentTable->rows.empty()) {
self->currentTable->rows.emplace_back();
@@ -695,7 +765,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Flush any pending text block so it appears before the image
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
if (self->currentTextBlock && !self->currentTextBlock->isEmpty()) {
const BlockStyle parentBlockStyle = self->currentTextBlock->getBlockStyle();
@@ -863,7 +933,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Flush buffer before style change
if (self->partWordBufferIndex > 0) {
const bool endsAtDashBreak = bufferEndsWithBreakableDash(self->partWordBuffer, self->partWordBufferIndex);
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
if (!endsAtDashBreak) {
self->nextWordContinues = true;
}
@@ -902,7 +972,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Otherwise tags like ..."item?"<p ...> can carry the final word into the next paragraph.
if (self->partWordBufferIndex > 0 && ((matches(name, HEADER_TAGS, NUM_HEADER_TAGS)) ||
(matches(name, BLOCK_TAGS, NUM_BLOCK_TAGS) && strcmp(name, "br") != 0))) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
if (matches(name, HEADER_TAGS, NUM_HEADER_TAGS)) {
@@ -935,7 +1005,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
if (strcmp(name, "br") == 0) {
if (self->partWordBufferIndex > 0) {
// flush word preceding <br/> to currentTextBlock before calling startNewTextBlock
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
// Tag the new block so startNewTextBlock can inject a full line-height gap if
// the block remains empty (i.e. <br> is a section separator between paragraphs).
@@ -981,7 +1051,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Flush buffer before style change so preceding text gets current style
if (self->partWordBufferIndex > 0) {
const bool endsAtDashBreak = bufferEndsWithBreakableDash(self->partWordBuffer, self->partWordBufferIndex);
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
if (!endsAtDashBreak) {
self->nextWordContinues = true;
}
@@ -1030,7 +1100,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Flush buffer before style change so preceding text gets current style
if (self->partWordBufferIndex > 0) {
const bool endsAtDashBreak = bufferEndsWithBreakableDash(self->partWordBuffer, self->partWordBufferIndex);
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
if (!endsAtDashBreak) {
self->nextWordContinues = true;
}
@@ -1062,7 +1132,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Flush buffer before style change so preceding text gets current style
if (self->partWordBufferIndex > 0) {
const bool endsAtDashBreak = bufferEndsWithBreakableDash(self->partWordBuffer, self->partWordBufferIndex);
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
if (!endsAtDashBreak) {
self->nextWordContinues = true;
}
@@ -1096,7 +1166,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
// Flush buffer before style change so preceding text gets current style
if (self->partWordBufferIndex > 0) {
const bool endsAtDashBreak = bufferEndsWithBreakableDash(self->partWordBuffer, self->partWordBufferIndex);
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
if (!endsAtDashBreak) {
self->nextWordContinues = true;
}
@@ -1141,6 +1211,10 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char* s, const int len) {
auto* self = static_cast<ChapterHtmlSlimParser*>(userData);
if (self->streamFailed) {
return;
}
// Skip content of nested tables (depth > 1 means we're inside a nested table)
if (self->currentTable && self->currentTable->depth > 1) {
return;
@@ -1203,7 +1277,7 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
if (self->partWordBufferIndex >= MAX_WORD_SIZE) {
// Buffer is full — flush before appending. Pure ASCII means no
// partial multi-byte sequence can be at the boundary.
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
self->partWordBuffer[self->partWordBufferIndex++] = s[i];
continue;
@@ -1213,7 +1287,7 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
// Inside <pre>: treat \n as a hard line break.
if (s[i] == '\n' && self->preUntilDepth < self->depth) {
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
// Blank line: the current block is empty, but we still need to emit a visible
// empty line. Add a single space so the block is non-empty and makePages()
@@ -1227,7 +1301,7 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
}
// Currently looking at whitespace, if there's anything in the partWordBuffer, flush it
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
// Whitespace is a real word boundary — reset continuation state
self->nextWordContinues = false;
@@ -1255,14 +1329,14 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
// "200 Quadrat-" / "kilometer" instead of the unusable "200" / "Quadratkilometer".
if (static_cast<uint8_t>(s[i]) == 0xC2 && i + 1 < len && static_cast<uint8_t>(s[i + 1]) == 0xA0) {
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
self->partWordBuffer[0] = ' ';
self->partWordBuffer[1] = '\0';
self->partWordBufferIndex = 1;
self->nextWordContinues = true; // Attach space to previous word (no break).
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
self->nextWordContinues = true; // Next real word attaches to this space (no break).
@@ -1274,14 +1348,14 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
if (static_cast<uint8_t>(s[i]) == 0xE2 && i + 2 < len && static_cast<uint8_t>(s[i + 1]) == 0x80 &&
static_cast<uint8_t>(s[i + 2]) == 0xAF) {
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
self->partWordBuffer[0] = ' ';
self->partWordBuffer[1] = '\0';
self->partWordBufferIndex = 1;
self->nextWordContinues = true;
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
self->nextWordContinues = true;
@@ -1319,13 +1393,13 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
saved[j] = self->partWordBuffer[safeLen + j];
}
self->partWordBufferIndex = safeLen;
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
for (int j = 0; j < overflow; j++) {
self->partWordBuffer[j] = saved[j];
}
self->partWordBufferIndex = overflow;
} else {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
}
@@ -1352,6 +1426,10 @@ void XMLCALL ChapterHtmlSlimParser::defaultHandlerExpand(void* userData, const X
void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* name) {
auto* self = static_cast<ChapterHtmlSlimParser*>(userData);
if (self->streamFailed) {
return;
}
// Check if any style state will change after we decrement depth
// If so, we MUST flush the partWordBuffer with the CURRENT style first
// Note: depth hasn't been decremented yet, so we check against (depth - 1)
@@ -1388,7 +1466,7 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n
if (shouldFlush) {
const bool endsAtDashBreak = bufferEndsWithBreakableDash(self->partWordBuffer, self->partWordBufferIndex);
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
// If closing an inline element, the next word fragment continues the same visual word —
// unless the buffered text ended at a dash that should allow a line break (em/en dash, etc.).
if (isInlineTag && !endsAtDashBreak) {
@@ -1431,7 +1509,7 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n
if (self->currentTable && self->currentTable->depth == 1 && (strcmp(name, "td") == 0 || strcmp(name, "th") == 0)) {
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
// Determine if the whole row consists of header cells
if (!self->currentTable->rows.empty()) {
@@ -1455,7 +1533,7 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n
if (self->currentTable && self->currentTable->depth == 1 && strcmp(name, "table") == 0) {
if (self->partWordBufferIndex > 0) {
self->flushPartWordBuffer();
if (!self->flushPartWordBuffer()) return;
}
self->currentTableCell = nullptr;
self->emitBufferedTable();
@@ -1560,6 +1638,7 @@ bool ChapterHtmlSlimParser::setup(const size_t totalInflatedSize) {
bytesStreamed = 0;
lastReportedProgress = -1;
streamFailed = false;
layoutFailed = false;
streamStartTimeMs = millis();
// Choose progress granularity by chapter size. Each callback drives a full-screen
@@ -1572,8 +1651,18 @@ bool ChapterHtmlSlimParser::setup(const size_t totalInflatedSize) {
progressStepPercent = 50;
}
const uint32_t popupFreeHeap = ESP.getFreeHeap();
const uint32_t popupContigHeap = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
progressUiEnabled =
popupFreeHeap >= MIN_FREE_HEAP_FOR_INDEXING_POPUP && popupContigHeap >= MIN_CONTIG_HEAP_FOR_INDEXING_POPUP;
if (!progressUiEnabled) {
LOG_DBG("EHP", "Skipping indexing popup due to low heap (free=%u contig=%u)", popupFreeHeap, popupContigHeap);
// When popup is disabled, also disable mid-parse ticks.
progressStepPercent = 0;
}
// Show initial progress popup for files above threshold.
if (progressFn && totalStreamSize >= MIN_SIZE_FOR_POPUP) {
if (progressFn && progressUiEnabled && totalStreamSize >= MIN_SIZE_FOR_POPUP) {
progressFn(0);
}
return true;
@@ -1614,7 +1703,7 @@ size_t ChapterHtmlSlimParser::write(const uint8_t* buffer, const size_t size) {
// Report progress at the granularity chosen up-front (see progressStepPercent).
// Skip the 100% callback — the page render that follows immediately replaces the popup,
// so the final tick is wasted work.
if (progressFn && progressStepPercent > 0 && totalStreamSize > 0) {
if (progressFn && progressUiEnabled && progressStepPercent > 0 && totalStreamSize > 0) {
const int progress = static_cast<int>(bytesStreamed * 100 / totalStreamSize);
if (progress < 100 && progress / progressStepPercent > lastReportedProgress / progressStepPercent) {
lastReportedProgress = progress;
@@ -1654,11 +1743,13 @@ bool ChapterHtmlSlimParser::finalize() {
// success scenario still flushes whatever pages were produced.
if (currentTextBlock) {
makePages();
if (!pendingAnchorId.empty()) {
anchorData.push_back({std::move(pendingAnchorId), static_cast<uint16_t>(completedPageCount)});
pendingAnchorId.clear();
if (!layoutFailed) {
if (!pendingAnchorId.empty()) {
anchorData.push_back({std::move(pendingAnchorId), static_cast<uint16_t>(completedPageCount)});
pendingAnchorId.clear();
}
emitPage(0u); // post-parse: no byte offset available
}
emitPage(0u); // post-parse: no byte offset available
currentPage.reset();
currentTextBlock.reset();
}
@@ -1706,6 +1797,11 @@ ParsedText::LineProcessResult ChapterHtmlSlimParser::addLineToPage(std::shared_p
}
void ChapterHtmlSlimParser::makePages() {
if (layoutFailed) {
currentTextBlock.reset();
return;
}
if (!currentTextBlock) {
LOG_ERR("EHP", "!! No text block to make pages for !!");
return;
@@ -1736,6 +1832,12 @@ void ChapterHtmlSlimParser::makePages() {
const uint16_t effectiveWidth =
(horizontalInset < viewportWidth) ? static_cast<uint16_t>(viewportWidth - horizontalInset) : viewportWidth;
if (!ensureHeapForTextLayout("paragraph layout")) {
layoutFailed = true;
currentTextBlock.reset();
return;
}
currentTextBlock->layoutAndExtractLines(
renderer, fontId, effectiveWidth,
[this](const std::shared_ptr<TextBlock>& textBlock, const bool lineEndsWithHyphenatedWord,
@@ -137,6 +137,7 @@ class ChapterHtmlSlimParser final : public Print {
size_t bytesStreamed = 0;
int lastReportedProgress = -1;
int progressStepPercent = 0;
bool progressUiEnabled = true;
bool streamFailed = false;
uint32_t streamStartTimeMs = 0;
@@ -148,6 +149,7 @@ class ChapterHtmlSlimParser final : public Print {
std::vector<std::pair<int, FootnoteEntry>> pendingFootnotes; // <wordIndex, entry>
int wordsExtractedInBlock = 0;
bool bionicReadingEnabled = false;
bool layoutFailed = false;
// Per-chapter caches: resolveStyle and parseInlineStyle are called for every HTML element;
// caching by (tag|classAttr) and styleAttr avoids repeated string operations and hash lookups.
@@ -155,8 +157,9 @@ class ChapterHtmlSlimParser final : public Print {
std::unordered_map<std::string, CssStyle> inlineStyleCache_;
void updateEffectiveInlineStyle();
bool ensureHeapForTextLayout(const char* phase);
void startNewTextBlock(const BlockStyle& blockStyle);
void flushPartWordBuffer();
bool flushPartWordBuffer();
void makePages();
void emitBufferedTable();
void emitTableAsFragments(BufferedTable& table);
+121 -10
View File
@@ -69,6 +69,9 @@ void GfxRenderer::begin() {
panelHeight = display.getDisplayHeight();
panelWidthBytes = display.getDisplayWidthBytes();
frameBufferSize = display.getBufferSize();
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
bwSnapshotSizeBytes = 0;
bwBufferChunkSize = BW_BUFFER_CHUNK_SIZE;
bwBufferChunks.assign((frameBufferSize + bwBufferChunkSize - 1) / bwBufferChunkSize, nullptr);
}
@@ -1780,9 +1783,11 @@ void GfxRenderer::fillPolygon(const int* xPoints, const int* yPoints, int numPoi
// For performance measurement (using static to allow "const" methods)
static unsigned long start_ms = 0;
static bool start_ms_valid = false;
void GfxRenderer::clearScreen(const uint8_t color) const {
start_ms = millis();
start_ms_valid = true;
display.clearScreen(color);
}
@@ -1818,8 +1823,13 @@ void GfxRenderer::displayBuffer(const HalDisplay::RefreshMode refreshMode) const
}
}
auto elapsed = millis() - start_ms;
LOG_DBG("GFX", "Time = %lu ms from clearScreen to displayBuffer", elapsed);
if (start_ms_valid) {
auto elapsed = millis() - start_ms;
LOG_DBG("GFX", "Time = %lu ms from clearScreen to displayBuffer", elapsed);
} else {
LOG_DBG("GFX", "Time = n/a from clearScreen to displayBuffer (no clearScreen marker)");
}
start_ms_valid = false;
display.displayBuffer(effectiveMode, fadingFix.load(std::memory_order_relaxed));
}
@@ -2143,9 +2153,91 @@ void GfxRenderer::freeBwBufferChunks() {
* Uses chunked allocation to avoid needing 48KB of contiguous memory.
* Returns true if buffer was stored successfully, false if allocation failed.
*/
bool GfxRenderer::storeBwBuffer() {
bool GfxRenderer::storeBwBuffer() { return storeBwBufferRect(0, 0, getScreenWidth(), getScreenHeight()); }
bool GfxRenderer::storeBwBufferRect(const int x, const int y, const int width, const int height) {
if (width <= 0 || height <= 0) {
freeBwBufferChunks();
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
bwSnapshotSizeBytes = 0;
LOG_ERR("GFX", "!! BW buffer store rect invalid: x=%d y=%d w=%d h=%d", x, y, width, height);
return false;
}
const int screenWidth = getScreenWidth();
const int screenHeight = getScreenHeight();
if (screenWidth <= 0 || screenHeight <= 0 || panelWidthBytes == 0 || panelHeight == 0 || !frameBuffer) {
freeBwBufferChunks();
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
bwSnapshotSizeBytes = 0;
LOG_ERR("GFX", "!! BW buffer store unavailable (screen=%dx%d panelHeight=%u rowBytes=%u fb=%p)", screenWidth,
screenHeight, panelHeight, panelWidthBytes, frameBuffer);
return false;
}
const int clampedX0 = std::max(0, x);
const int clampedY0 = std::max(0, y);
const int clampedX1 = std::min(screenWidth - 1, x + width - 1);
const int clampedY1 = std::min(screenHeight - 1, y + height - 1);
if (clampedX0 > clampedX1 || clampedY0 > clampedY1) {
freeBwBufferChunks();
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
bwSnapshotSizeBytes = 0;
LOG_ERR("GFX", "!! BW buffer store rect outside screen: x=%d y=%d w=%d h=%d", x, y, width, height);
return false;
}
int rowStart = 0;
int rowEnd = 0;
switch (getOrientation()) {
case LandscapeCounterClockwise:
rowStart = clampedY0;
rowEnd = clampedY1;
break;
case LandscapeClockwise:
rowStart = static_cast<int>(panelHeight) - 1 - clampedY1;
rowEnd = static_cast<int>(panelHeight) - 1 - clampedY0;
break;
case Portrait:
rowStart = static_cast<int>(panelHeight) - 1 - clampedX1;
rowEnd = static_cast<int>(panelHeight) - 1 - clampedX0;
break;
case PortraitInverted:
rowStart = clampedX0;
rowEnd = clampedX1;
break;
}
rowStart = std::max(0, rowStart);
rowEnd = std::min(static_cast<int>(panelHeight) - 1, rowEnd);
if (rowStart > rowEnd) {
freeBwBufferChunks();
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
bwSnapshotSizeBytes = 0;
LOG_ERR("GFX", "!! BW buffer store row-band invalid after orientation mapping: rows=%d..%d", rowStart, rowEnd);
return false;
}
const size_t rows = static_cast<size_t>(rowEnd - rowStart + 1);
const size_t snapshotSizeBytes = rows * panelWidthBytes;
const size_t snapshotBaseOffset = static_cast<size_t>(rowStart) * panelWidthBytes;
if (snapshotSizeBytes == 0 || snapshotBaseOffset + snapshotSizeBytes > frameBufferSize) {
LOG_ERR("GFX", "!! BW buffer store row-band out of bounds: base=%zu size=%zu frame=%u", snapshotBaseOffset,
snapshotSizeBytes, frameBufferSize);
return false;
}
freeBwBufferChunks();
bwSnapshotRowStart = static_cast<uint16_t>(rowStart);
bwSnapshotRowEnd = static_cast<uint16_t>(rowEnd);
bwSnapshotSizeBytes = snapshotSizeBytes;
auto attemptStore = [&](size_t chunkSize) {
bwBufferChunks.assign((frameBufferSize + chunkSize - 1) / chunkSize, nullptr);
bwBufferChunks.assign((bwSnapshotSizeBytes + chunkSize - 1) / chunkSize, nullptr);
for (size_t i = 0; i < bwBufferChunks.size(); i++) {
if (bwBufferChunks[i]) {
LOG_ERR("GFX", "!! BW buffer chunk %zu already stored - this is likely a bug, freeing chunk", i);
@@ -2154,7 +2246,7 @@ bool GfxRenderer::storeBwBuffer() {
}
const size_t offset = i * chunkSize;
const size_t allocSize = std::min(chunkSize, static_cast<size_t>(frameBufferSize - offset));
const size_t allocSize = std::min(chunkSize, bwSnapshotSizeBytes - offset);
bwBufferChunks[i] = static_cast<uint8_t*>(malloc(allocSize));
if (!bwBufferChunks[i]) {
@@ -2166,10 +2258,11 @@ bool GfxRenderer::storeBwBuffer() {
return false;
}
memcpy(bwBufferChunks[i], frameBuffer + offset, allocSize);
memcpy(bwBufferChunks[i], frameBuffer + snapshotBaseOffset + offset, allocSize);
}
bwBufferChunkSize = chunkSize;
LOG_DBG("GFX", "Stored BW buffer in %zu chunks (%zu bytes each)", bwBufferChunks.size(), chunkSize);
LOG_DBG("GFX", "Stored BW buffer rows [%u..%u] (%zu bytes) in %zu chunks (%zu bytes each)", bwSnapshotRowStart,
bwSnapshotRowEnd, bwSnapshotSizeBytes, bwBufferChunks.size(), chunkSize);
return true;
};
@@ -2199,6 +2292,9 @@ bool GfxRenderer::storeBwBuffer() {
}
LOG_ERR("GFX", "!! BW buffer storage failed after retrying smaller chunk sizes");
bwSnapshotSizeBytes = 0;
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
return false;
}
@@ -2208,6 +2304,13 @@ bool GfxRenderer::storeBwBuffer() {
* Uses chunked restoration to match chunked storage.
*/
void GfxRenderer::restoreBwBuffer() {
if (bwSnapshotSizeBytes == 0) {
display.cleanupGrayscaleBuffers(frameBuffer);
freeBwBufferChunks();
LOG_ERR("GFX", "BW restore skipped: no stored snapshot metadata; cleaned grayscale buffers only");
return;
}
// Check if all chunks are allocated
bool missingChunks = false;
for (const auto& bwBufferChunk : bwBufferChunks) {
@@ -2223,20 +2326,28 @@ void GfxRenderer::restoreBwBuffer() {
// allocations that can later starve TLS handshakes.
display.cleanupGrayscaleBuffers(frameBuffer);
freeBwBufferChunks();
bwSnapshotSizeBytes = 0;
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
LOG_ERR("GFX", "BW restore skipped due to missing chunks; cleaned grayscale buffers only");
return;
}
const size_t snapshotBaseOffset = static_cast<size_t>(bwSnapshotRowStart) * panelWidthBytes;
for (size_t i = 0; i < bwBufferChunks.size(); i++) {
const size_t offset = i * bwBufferChunkSize;
const size_t chunkSize = std::min(bwBufferChunkSize, static_cast<size_t>(frameBufferSize - offset));
memcpy(frameBuffer + offset, bwBufferChunks[i], chunkSize);
const size_t chunkSize = std::min(bwBufferChunkSize, bwSnapshotSizeBytes - offset);
memcpy(frameBuffer + snapshotBaseOffset + offset, bwBufferChunks[i], chunkSize);
}
display.cleanupGrayscaleBuffers(frameBuffer);
freeBwBufferChunks();
LOG_DBG("GFX", "Restored and freed BW buffer chunks");
LOG_DBG("GFX", "Restored BW buffer rows [%u..%u] (%zu bytes) and freed BW chunks", bwSnapshotRowStart,
bwSnapshotRowEnd, bwSnapshotSizeBytes);
bwSnapshotSizeBytes = 0;
bwSnapshotRowStart = 0;
bwSnapshotRowEnd = 0;
}
/**
+6 -2
View File
@@ -55,6 +55,9 @@ class GfxRenderer {
uint16_t panelHeight = 0; // set in begin()
uint16_t panelWidthBytes = 0; // set in begin()
uint32_t frameBufferSize = 0; // set in begin()
uint16_t bwSnapshotRowStart = 0;
uint16_t bwSnapshotRowEnd = 0;
size_t bwSnapshotSizeBytes = 0;
size_t bwBufferChunkSize = BW_BUFFER_CHUNK_SIZE;
std::vector<uint8_t*> bwBufferChunks;
std::map<int, EpdFontFamily> fontMap;
@@ -206,8 +209,9 @@ class GfxRenderer {
void copyGrayscaleLsbBuffers() const;
void copyGrayscaleMsbBuffers() const;
void displayGrayBuffer() const;
bool storeBwBuffer(); // Returns true if buffer was stored successfully
void restoreBwBuffer(); // Restore and free the stored buffer
bool storeBwBuffer(); // Returns true if buffer was stored successfully
bool storeBwBufferRect(int x, int y, int width, int height); // Store only rows intersecting logical rect
void restoreBwBuffer(); // Restore and free the stored buffer
void cleanupGrayscaleWithFrameBuffer() const;
// Font helpers
+200 -12
View File
@@ -44,6 +44,50 @@ constexpr unsigned long skipChapterMs = 700;
// pages per minute, first item is 1 to prevent division by zero if accessed
constexpr int PAGE_TURN_LABELS[] = {1, 1, 3, 6, 12};
// Silent next-chapter indexing should only run when heap is healthy enough,
// otherwise it tends to produce heavily truncated fallback caches.
#ifndef CP_SILENT_INDEX_MIN_FREE_HEAP_BYTES
#define CP_SILENT_INDEX_MIN_FREE_HEAP_BYTES (64 * 1024)
#endif
#ifndef CP_SILENT_INDEX_MIN_CONTIG_HEAP_BYTES
#define CP_SILENT_INDEX_MIN_CONTIG_HEAP_BYTES (24 * 1024)
#endif
constexpr uint32_t SILENT_INDEX_MIN_FREE_HEAP_BYTES = CP_SILENT_INDEX_MIN_FREE_HEAP_BYTES;
constexpr uint32_t SILENT_INDEX_MIN_CONTIG_HEAP_BYTES = CP_SILENT_INDEX_MIN_CONTIG_HEAP_BYTES;
// Hysteresis for auto AA recovery after low-memory BW snapshot failures.
// Override at build time via -D flags, for example:
// -DCP_AA_RECOVERY_FREE_HEAP_BYTES=30720
// -DCP_AA_RECOVERY_CONTIG_HEAP_BYTES=12288
#ifndef CP_AA_RECOVERY_FREE_HEAP_BYTES
#define CP_AA_RECOVERY_FREE_HEAP_BYTES (28 * 1024)
#endif
#ifndef CP_AA_RECOVERY_CONTIG_HEAP_BYTES
#define CP_AA_RECOVERY_CONTIG_HEAP_BYTES (12 * 1024)
#endif
constexpr uint32_t AA_RECOVERY_FREE_HEAP_BYTES = CP_AA_RECOVERY_FREE_HEAP_BYTES;
constexpr uint32_t AA_RECOVERY_CONTIG_HEAP_BYTES = CP_AA_RECOVERY_CONTIG_HEAP_BYTES;
// Snapshotting BW buffer for grayscale restore can fragment heap heavily on tight
// pages. Skip attempting snapshot altogether below this safety window.
#ifndef CP_BW_SNAPSHOT_MIN_FREE_HEAP_BYTES
#define CP_BW_SNAPSHOT_MIN_FREE_HEAP_BYTES (64 * 1024)
#endif
#ifndef CP_BW_SNAPSHOT_MIN_CONTIG_HEAP_BYTES
#define CP_BW_SNAPSHOT_MIN_CONTIG_HEAP_BYTES (24 * 1024)
#endif
constexpr uint32_t BW_SNAPSHOT_MIN_FREE_HEAP_BYTES = CP_BW_SNAPSHOT_MIN_FREE_HEAP_BYTES;
constexpr uint32_t BW_SNAPSHOT_MIN_CONTIG_HEAP_BYTES = CP_BW_SNAPSHOT_MIN_CONTIG_HEAP_BYTES;
constexpr uint8_t TRUNCATED_SECTION_HINT_RENDER_COUNT = 2;
constexpr const char* TRUNCATED_SECTION_HINT_LINE_1 = "Chapter may be truncated (low memory).";
constexpr const char* TRUNCATED_SECTION_HINT_LINE_2 = "Try: No embedded style | No images | AA Off";
#if DEBUG_MEMORY_CONSUMPTION
void logReaderMemSnapshot(const char* stage) {
const uint32_t freeHeap = esp_get_free_heap_size();
@@ -54,6 +98,62 @@ void logReaderMemSnapshot(const char* stage) {
inline void logReaderMemSnapshot(const char*) {}
#endif
bool computePageDynamicYBand(const Page& page, const GfxRenderer& renderer, const int fontId, const int viewportHeight,
int* outTop, int* outBottom) {
if (viewportHeight <= 0 || !outTop || !outBottom) {
return false;
}
bool hasRange = false;
int minY = viewportHeight;
int maxY = -1;
const int lineHeight = std::max(1, renderer.getLineHeight(fontId));
for (const auto& el : page.elements) {
if (!el) continue;
const int elementTop = el->yPos;
int elementBottom = elementTop;
switch (el->getTag()) {
case TAG_PageLine:
elementBottom = el->yPos + lineHeight;
break;
case TAG_PageImage: {
const auto& img = static_cast<const PageImage&>(*el);
elementBottom = el->yPos + img.getImageBlock().getHeight();
break;
}
case TAG_PageTable: {
const auto& table = static_cast<const PageTableFragment&>(*el);
elementBottom = el->yPos + table.getTotalHeight();
break;
}
default:
continue;
}
minY = std::min(minY, elementTop);
maxY = std::max(maxY, elementBottom);
hasRange = true;
}
if (!hasRange) {
return false;
}
constexpr int BAND_PAD_PX = 2;
minY = std::max(0, minY - BAND_PAD_PX);
maxY = std::min(viewportHeight, maxY + BAND_PAD_PX);
if (minY >= maxY) {
return false;
}
*outTop = minY;
*outBottom = maxY;
return true;
}
// Computes the [0..100] EPUB progress percent. Returns 0 when pageCount is unknown (sync/bookmark
// pre-render writes), in which case the next saveProgress() will overwrite progress.bin with the
// real value before the user can leave the reader.
@@ -1300,6 +1400,7 @@ void EpubReaderActivity::render(RenderLock&& lock) {
lastRenderStats.textAntiAliasing = SETTINGS.textAntiAliasing;
lastRenderStats.freeHeapBefore = esp_get_free_heap_size();
lastRenderStats.largestFreeBlockBefore = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
showTruncatedSectionHintThisRender = false;
if (!section) {
if (currentSpineIndex < 0 || currentSpineIndex >= spineCount) {
@@ -1353,6 +1454,12 @@ void EpubReaderActivity::render(RenderLock&& lock) {
}
lastRenderStats.sectionLoadMs = millis() - sectionStart;
if (section->isTruncatedCache() && currentSpineIndex != lastWarnedTruncatedSpineIndex) {
lastWarnedTruncatedSpineIndex = currentSpineIndex;
truncatedSectionHintRendersRemaining = TRUNCATED_SECTION_HINT_RENDER_COUNT;
LOG_INF("ERS", "Section %d is truncated; showing mitigation hint", currentSpineIndex);
}
if (nextPageNumber == UINT16_MAX) {
section->currentPage = section->pageCount - 1;
} else {
@@ -1472,10 +1579,14 @@ void EpubReaderActivity::render(RenderLock&& lock) {
lastRenderStats.spineIndex = currentSpineIndex;
lastRenderStats.pageIndex = section->currentPage;
lastRenderStats.pageCount = section->pageCount;
showTruncatedSectionHintThisRender = truncatedSectionHintRendersRemaining > 0;
const auto start = millis();
renderContents(std::move(p), orientedMarginTop, orientedMarginRight, orientedMarginBottom, orientedMarginLeft);
lastRenderStats.requestRenderMs = millis() - start;
if (truncatedSectionHintRendersRemaining > 0) {
truncatedSectionHintRendersRemaining--;
}
LOG_DBG("ERS", "Rendered page in %dms", lastRenderStats.requestRenderMs);
}
silentIndexNextChapterIfNeeded(viewportWidth, viewportHeight);
@@ -1495,6 +1606,13 @@ void EpubReaderActivity::silentIndexNextChapterIfNeeded(const uint16_t viewportW
return;
}
const uint32_t freeHeap = esp_get_free_heap_size();
const uint32_t contigHeap = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
if (freeHeap < SILENT_INDEX_MIN_FREE_HEAP_BYTES || contigHeap < SILENT_INDEX_MIN_CONTIG_HEAP_BYTES) {
LOG_DBG("ERS", "Skipping silent indexing due to low heap (free=%lu contig=%lu)", freeHeap, contigHeap);
return;
}
// Build the next chapter cache while the penultimate page is on screen.
if (section->currentPage != section->pageCount - 2) {
return;
@@ -1560,8 +1678,23 @@ void EpubReaderActivity::renderContents(std::unique_ptr<Page> page, const int or
(int32_t)heapAfter - (int32_t)heapBefore);
logReaderMemSnapshot("prewarm_end");
const bool aaConfigured = SETTINGS.textAntiAliasing;
bool aaEnabledForThisRender = aaConfigured;
if (aaConfigured && antiAliasingSuspendedLowMemory) {
const uint32_t freeHeapNow = esp_get_free_heap_size();
const uint32_t contigHeapNow = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
if (freeHeapNow >= AA_RECOVERY_FREE_HEAP_BYTES && contigHeapNow >= AA_RECOVERY_CONTIG_HEAP_BYTES) {
antiAliasingSuspendedLowMemory = false;
LOG_INF("ERS", "Re-enabling text anti-aliasing after heap recovery (free=%lu contig=%lu)", freeHeapNow,
contigHeapNow);
} else {
aaEnabledForThisRender = false;
}
}
lastRenderStats.textAntiAliasing = aaEnabledForThisRender;
// Force special handling for pages with images when anti-aliasing is on
bool imagePageWithAA = page->hasImages() && SETTINGS.textAntiAliasing;
bool imagePageWithAA = page->hasImages() && aaEnabledForThisRender;
bool forceHalfRefreshThisPage = pendingHalfRefreshAfterImagePage && SETTINGS.halfRefreshAfterImagePage;
pendingHalfRefreshAfterImagePage = false;
lastRenderStats.imagePageWithAA = imagePageWithAA;
@@ -1570,6 +1703,23 @@ void EpubReaderActivity::renderContents(std::unique_ptr<Page> page, const int or
logReaderMemSnapshot("before_bw_render");
page->render(renderer, getEffectiveReaderFontId(), orientedMarginLeft, contentTop);
renderStatusBar();
if (showTruncatedSectionHintThisRender) {
const int hintX = orientedMarginLeft + 4;
const int hintY1 = contentTop + 4;
const int hintY2 = hintY1 + 20;
const int maxWidth = std::max(0, renderer.getScreenWidth() - orientedMarginLeft - orientedMarginRight - 8);
// Clear a dedicated band so the hint stays readable over any page content.
const int boxX = hintX - 2;
const int boxY = hintY1 - 2;
const int boxW = maxWidth + 4;
const int boxH = 44;
renderer.fillRect(boxX, boxY, boxW, boxH, false);
renderer.drawRect(boxX, boxY, boxW, boxH, true);
const std::string line1 = renderer.truncatedText(UI_10_FONT_ID, TRUNCATED_SECTION_HINT_LINE_1, maxWidth);
const std::string line2 = renderer.truncatedText(UI_10_FONT_ID, TRUNCATED_SECTION_HINT_LINE_2, maxWidth);
renderer.drawText(UI_10_FONT_ID, hintX, hintY1, line1.c_str(), true, EpdFontFamily::BOLD);
renderer.drawText(UI_10_FONT_ID, hintX, hintY2, line2.c_str(), true);
}
fcm->logStats("bw_render");
const auto tBwRender = millis();
logReaderMemSnapshot("after_bw_render");
@@ -1604,11 +1754,46 @@ void EpubReaderActivity::renderContents(std::unique_ptr<Page> page, const int or
}
const auto tDisplay = millis();
// Save bw buffer to reset buffer state after grayscale data sync
// Save bw buffer to reset buffer state after grayscale data sync.
// Pre-check heap to avoid entering the chunk-retry path when success is unlikely.
const uint32_t bwStoreFreeHeap = esp_get_free_heap_size();
const uint32_t bwStoreContigHeap = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
const bool shouldAttemptBwSnapshot =
bwStoreFreeHeap >= BW_SNAPSHOT_MIN_FREE_HEAP_BYTES && bwStoreContigHeap >= BW_SNAPSHOT_MIN_CONTIG_HEAP_BYTES;
logReaderMemSnapshot("bw_store_begin");
renderer.storeBwBuffer();
bool bwBufferStored = false;
if (shouldAttemptBwSnapshot) {
const int contentLeft = orientedMarginLeft;
const int contentRight = std::max(contentLeft, renderer.getScreenWidth() - orientedMarginRight);
const int contentBottom = std::max(contentTop, renderer.getScreenHeight() - orientedMarginBottom);
int bandTop = 0;
int bandBottom = std::max(0, contentBottom - contentTop);
if (!computePageDynamicYBand(*page, renderer, getEffectiveReaderFontId(), bandBottom, &bandTop, &bandBottom)) {
bandTop = 0;
bandBottom = std::max(0, contentBottom - contentTop);
}
const int snapshotTop = contentTop + bandTop;
const int snapshotHeight = std::max(0, bandBottom - bandTop);
bwBufferStored = renderer.storeBwBufferRect(contentLeft, snapshotTop, contentRight - contentLeft, snapshotHeight);
} else {
LOG_INF("ERS", "Skipping BW snapshot precheck (free=%lu contig=%lu, need free>=%lu contig>=%lu)", bwStoreFreeHeap,
bwStoreContigHeap, static_cast<uint32_t>(BW_SNAPSHOT_MIN_FREE_HEAP_BYTES),
static_cast<uint32_t>(BW_SNAPSHOT_MIN_CONTIG_HEAP_BYTES));
}
const auto tBwStore = millis();
logReaderMemSnapshot("bw_store_end");
if (!bwBufferStored) {
if (aaEnabledForThisRender && !antiAliasingSuspendedLowMemory) {
antiAliasingSuspendedLowMemory = true;
const uint32_t freeHeapNow = esp_get_free_heap_size();
const uint32_t contigHeapNow = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT | MALLOC_CAP_DEFAULT);
LOG_INF("ERS", "Suspending text anti-aliasing due to low heap (free=%lu contig=%lu)", freeHeapNow, contigHeapNow);
}
LOG_INF("ERS", "Skipping grayscale/BW-restore for this page (insufficient heap for BW snapshot)");
}
if (page->hasImages() && getEffectiveImageRendering() != CrossPointSettings::IMAGES_SUPPRESS) {
pendingHalfRefreshAfterImagePage = true;
@@ -1616,7 +1801,7 @@ void EpubReaderActivity::renderContents(std::unique_ptr<Page> page, const int or
// grayscale rendering
// TODO: Only do this if font supports it
if (SETTINGS.textAntiAliasing) {
if (aaEnabledForThisRender && bwBufferStored) {
logReaderMemSnapshot("gray_lsb_begin");
renderer.clearScreen(0x00);
renderer.setRenderMode(GfxRenderer::GRAYSCALE_LSB);
@@ -1659,21 +1844,24 @@ void EpubReaderActivity::renderContents(std::unique_ptr<Page> page, const int or
tPrewarm - t0, tBwRender - tPrewarm, tDisplay - tBwRender, tBwStore - tDisplay, tGrayLsb - tBwStore,
tGrayMsb - tGrayLsb, tGrayDisplay - tGrayMsb, tBwRestore - tGrayDisplay, tEnd - t0);
} else {
// restore the bw data
logReaderMemSnapshot("bw_restore_begin");
renderer.restoreBwBuffer();
const auto tBwRestore = millis();
logReaderMemSnapshot("bw_restore_end");
uint32_t bwRestoreMs = 0;
if (bwBufferStored) {
// restore the bw data
logReaderMemSnapshot("bw_restore_begin");
renderer.restoreBwBuffer();
const auto tBwRestore = millis();
logReaderMemSnapshot("bw_restore_end");
bwRestoreMs = tBwRestore - tBwStore;
}
const auto tEnd = millis();
lastRenderStats.usedGrayscale = false;
lastRenderStats.phases = {
tPrewarm - t0, tBwRender - tPrewarm, tDisplay - tBwRender, tBwStore - tDisplay, 0, 0, 0, tBwRestore - tBwStore,
tPrewarm - t0, tBwRender - tPrewarm, tDisplay - tBwRender, tBwStore - tDisplay, 0, 0, 0, bwRestoreMs,
tEnd - t0};
LOG_DBG("ERS",
"Page render: prewarm=%lums bw_render=%lums display=%lums bw_store=%lums bw_restore=%lums total=%lums",
tPrewarm - t0, tBwRender - tPrewarm, tDisplay - tBwRender, tBwStore - tDisplay, tBwRestore - tBwStore,
tEnd - t0);
tPrewarm - t0, tBwRender - tPrewarm, tDisplay - tBwRender, tBwStore - tDisplay, bwRestoreMs, tEnd - t0);
}
if (const auto* cacheManager = renderer.getFontCacheManager()) {
@@ -40,6 +40,12 @@ class EpubReaderActivity final : public Activity {
unsigned long lastPageTurnTime = 0UL;
unsigned long pageTurnDuration = 0UL;
bool pendingHalfRefreshAfterImagePage = false;
// Temporary AA suspension when BW snapshot allocation fails under memory pressure.
// Automatically lifted once heap recovers above hysteresis thresholds.
bool antiAliasingSuspendedLowMemory = false;
bool showTruncatedSectionHintThisRender = false;
uint8_t truncatedSectionHintRendersRemaining = 0;
int lastWarnedTruncatedSpineIndex = -1;
struct RenderPhaseStats {
unsigned long prewarmMs = 0UL;
unsigned long bwRenderMs = 0UL;