eric/Crosspoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove page cache, proper 64 bit filesize and apply upstream 1648

Browse Source
This commit is contained in:
jpirnay
2026-04-15 17:20:36 +02:00
parent a5dd44b3e0
commit 201aef565b
8 changed files with 216 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Xtc/Xtc.cpp
+3 -19
View File
@@ -14,14 +14,11 @@
bool Xtc::load() {
LOG_DBG("XTC", "Loading XTC: %s", filepath.c_str());
// Ensure the per-book cache exists before the parser tries to create page_table.bin.
setupCacheDir();
// Initialize parser
parser.reset(new xtc::XtcParser());
// Open XTC file and initialize its cache-backed page table
xtc::XtcError err = parser->open(filepath.c_str(), cachePath.c_str());
// Open XTC file
xtc::XtcError err = parser->open(filepath.c_str());
if (err != xtc::XtcError::OK) {
LOG_ERR("XTC", "Failed to load: %s", xtc::errorToString(err));
parser.reset();
@@ -106,7 +103,7 @@ bool Xtc::hasChapters() const {
return parser->hasChapters();
}
const std::vector<xtc::ChapterInfo>& Xtc::getChapters() const {
const std::vector<xtc::ChapterInfo>& Xtc::getChapters() {
static const std::vector<xtc::ChapterInfo> kEmpty;
if (!loaded || !parser) {
return kEmpty;
@@ -202,7 +199,6 @@ bool Xtc::generateCoverBmp() const {
uint8_t* rowBuffer = static_cast<uint8_t*>(malloc(dstRowSize));
if (!rowBuffer) {
free(pageBuffer);
coverBmp.close();
return false;
}
@@ -258,7 +254,6 @@ bool Xtc::generateCoverBmp() const {
}
}
coverBmp.close();
free(pageBuffer);
LOG_DBG("XTC", "Generated cover BMP: %s", getCoverBmpPath().c_str());
@@ -319,9 +314,7 @@ bool Xtc::generateThumbBmp(int height) const {
size_t bytesRead = src.read(buffer, sizeof(buffer));
dst.write(buffer, bytesRead);
}
dst.close();
}
src.close();
}
LOG_DBG("XTC", "Copied cover to thumb (no scaling needed)");
return Storage.exists(getThumbBmpPath(height).c_str());
@@ -375,7 +368,6 @@ bool Xtc::generateThumbBmp(int height) const {
uint8_t* rowBuffer = static_cast<uint8_t*>(malloc(rowSize));
if (!rowBuffer) {
free(pageBuffer);
thumbBmp.close();
return false;
}
@@ -482,7 +474,6 @@ bool Xtc::generateThumbBmp(int height) const {
}
free(rowBuffer);
thumbBmp.close();
free(pageBuffer);
LOG_DBG("XTC", "Generated thumb BMP (%dx%d): %s", thumbWidth, thumbHeight, getThumbBmpPath(height).c_str());
@@ -545,11 +536,4 @@ xtc::XtcError Xtc::getLastError() const {
return xtc::XtcError::FILE_NOT_FOUND;
}
return parser->getLastError();
}
void Xtc::prefetchPages(uint32_t pageIndex) const {
if (!loaded || !parser) {
return;
}
parser->prefetchWindow(pageIndex);
}
lib/Xtc/Xtc.h
+2 -7
View File
@@ -50,11 +50,6 @@ class Xtc {
*/
void setupCacheDir() const;
/**
* Preload window around specified page (for page turn optimization)
*/
void prefetchPages(uint32_t pageIndex) const;
// Path accessors
const std::string& getCachePath() const { return cachePath; }
const std::string& getPath() const { return filepath; }
@@ -63,7 +58,7 @@ class Xtc {
std::string getTitle() const;
std::string getAuthor() const;
bool hasChapters() const;
const std::vector<xtc::ChapterInfo>& getChapters() const;
const std::vector<xtc::ChapterInfo>& getChapters();
// Cover image support (for sleep screen)
std::string getCoverBmpPath() const;
@@ -107,4 +102,4 @@ class Xtc {
// Error information
xtc::XtcError getLastError() const;
};
};
lib/Xtc/Xtc/XtcParser.cpp
+164 -514
View File
@@ -10,80 +10,33 @@
#include <FsHelpers.h>
#include <HalStorage.h>
#include <Logging.h>
#include <esp_heap_caps.h>
#include <cstring>
#include <limits>
namespace xtc {
namespace {
constexpr size_t MAX_CHAPTERS = 4096;
bool canSeekToOffset(const uint64_t offset) {
return offset <= static_cast<uint64_t>(std::numeric_limits<size_t>::max());
}
bool seekToOffset(FsFile& file, const uint64_t offset) {
if (!canSeekToOffset(offset)) {
return false;
}
return file.seek(static_cast<size_t>(offset));
}
} // namespace
void XtcParser::safeDeserializeHeader(const uint8_t* buf, PageTableCacheHeader& header) {
memcpy(&header.magic, buf + 0, 4);
memcpy(&header.version, buf + 4, 4);
memcpy(&header.pageCount, buf + 8, 4);
memcpy(&header.originalHash, buf + 12, 4);
memcpy(&header.originalSize, buf + 16, 8);
memcpy(&header.entrySize, buf + 24, 4);
memcpy(&header.reserved, buf + 28, 4);
}
void XtcParser::safeSerializeHeader(uint8_t* buf, const PageTableCacheHeader& header) {
memcpy(buf + 0, &header.magic, 4);
memcpy(buf + 4, &header.version, 4);
memcpy(buf + 8, &header.pageCount, 4);
memcpy(buf + 12, &header.originalHash, 4);
memcpy(buf + 16, &header.originalSize, 8);
memcpy(buf + 24, &header.entrySize, 4);
memcpy(buf + 28, &header.reserved, 4);
}
XtcParser::XtcParser()
: m_isOpen(false),
m_defaultWidth(DISPLAY_WIDTH),
m_defaultHeight(DISPLAY_HEIGHT),
m_bitDepth(1),
m_hasChapters(false),
m_chaptersLoaded(false),
m_lastError(XtcError::OK) {
memset(&m_header, 0, sizeof(m_header));
for (auto& entry : m_l1Cache) {
entry.pageIndex = 0xFFFFFFFF;
entry.lastAccess = 0;
}
}
XtcParser::~XtcParser() { close(); }
XtcError XtcParser::open(const char* filepath, const char* cacheDir) {
// Close any previous file state before reopening
XtcError XtcParser::open(const char* filepath) {
// Close if already open
if (m_isOpen) {
close();
}
m_originalPath = filepath;
m_cacheDir = cacheDir;
m_filepath = filepath;
uint32_t fileHash = calculateFileHash(filepath);
m_cacheFilePath = std::string(cacheDir) + "/xtc_" + std::to_string(fileHash) + "/page_table.bin";
// Open the original XTC file just long enough to read metadata and validate the header.
// Open file
if (!Storage.openFileForRead("XTC", filepath, m_file)) {
m_lastError = XtcError::FILE_NOT_FOUND;
return m_lastError;
@@ -93,420 +46,82 @@ XtcError XtcParser::open(const char* filepath, const char* cacheDir) {
m_lastError = readHeader();
if (m_lastError != XtcError::OK) {
LOG_DBG("XTC", "Failed to read header: %s", errorToString(m_lastError));
// Explicit close() required: member variable persists beyond function scope
m_file.close();
return m_lastError;
}
if (m_header.pageCount == 0) {
LOG_ERR("XTC", "File has no pages");
m_file.close();
m_lastError = XtcError::CORRUPTED_HEADER;
return m_lastError;
}
// Metadata strings are small, so keep them in memory even when the page table is moved to cache.
// Read title & author if available
if (m_header.hasMetadata) {
readTitle();
readAuthor();
m_title.shrink_to_fit();
m_author.shrink_to_fit();
LOG_INF("XTC", "Metadata strings: titleLen=%u cap=%u, authorLen=%u cap=%u",
static_cast<unsigned int>(m_title.size()), static_cast<unsigned int>(m_title.capacity()),
static_cast<unsigned int>(m_author.size()), static_cast<unsigned int>(m_author.capacity()));
}
// Defer chapter parsing until the reader actually needs the table of contents.
m_pageTableOffset = m_header.pageTableOffset;
m_hasChapters = (m_header.hasChapters == 1) && (m_header.chapterOffset != 0);
LOG_INF("XTC", "Chapter metadata deferred: available=%s", m_hasChapters ? "yes" : "no");
m_file.close();
// Build or reuse the on-disk page table cache before marking the parser open.
if (!isPageTableCacheValid()) {
LOG_INF("XTC", "Building page table cache for %u pages", m_header.pageCount);
m_lastError = buildPageTableCache();
m_lastError = readTitle();
if (m_lastError != XtcError::OK) {
LOG_ERR("XTC", "Failed to build page table cache");
LOG_DBG("XTC", "Failed to read title: %s", errorToString(m_lastError));
// Explicit close() required: member variable persists beyond function scope
m_file.close();
return m_lastError;
}
const size_t heapBefore = ESP.getMaxAllocHeap();
LOG_DBG("XTC", "Cache built, heap before defrag: free=%zu, maxAlloc=%zu", ESP.getFreeHeap(), heapBefore);
// Defragment heap: small delay allows heap coalescing after file handles are closed
// This typically improves MaxAlloc by 10-20KB, enabling 96KB page buffer for grayscale
LOG_DBG("XTC", "Defragmenting heap (waiting 50ms)...");
vTaskDelay(pdMS_TO_TICKS(50));
const size_t heapAfter = ESP.getMaxAllocHeap();
const size_t heapGain = heapAfter > heapBefore ? (heapAfter - heapBefore) : 0;
if (heapGain > 0) {
LOG_INF("XTC", "Heap defragmented: +%zu bytes contiguous (now %zu)", heapGain, heapAfter);
} else {
LOG_DBG("XTC", "Heap after defrag: free=%zu, maxAlloc=%zu", ESP.getFreeHeap(), heapAfter);
m_lastError = readAuthor();
if (m_lastError != XtcError::OK) {
LOG_DBG("XTC", "Failed to read author: %s", errorToString(m_lastError));
// Explicit close() required: member variable persists beyond function scope
m_file.close();
return m_lastError;
}
// Trim excess capacity from metadata strings
m_title.shrink_to_fit();
m_author.shrink_to_fit();
}
if (!openCacheFile()) {
LOG_ERR("XTC", "Failed to open cache file");
m_lastError = XtcError::FILE_NOT_FOUND;
// Read first page info for default dimensions (no bulk page table allocation)
m_lastError = readFirstPageInfo();
if (m_lastError != XtcError::OK) {
LOG_DBG("XTC", "Failed to read first page info: %s", errorToString(m_lastError));
// Explicit close() required: member variable persists beyond function scope
m_file.close();
return m_lastError;
}
// Prime the sliding L2 window with the first chunk of page metadata.
loadL2Window(0);
// Defer chapter parsing until actually needed (lazy load).
// Chapter strings can use significant heap; keeping them out of memory
// during rendering leaves more room for the page bitmap buffer.
m_hasChapters = (m_header.hasChapters == 1);
m_chaptersLoaded = false;
LOG_DBG("XTC", "File opened, heap: free=%zu, maxAlloc=%zu", ESP.getFreeHeap(), ESP.getMaxAllocHeap());
// Close the source file to free its internal SdFat buffers.
// It will be reopened on-demand for page table lookups and bitmap reads.
m_file.close();
m_isOpen = true;
LOG_DBG("XTC", "Opened file: %s (%u pages, cache: %s)", filepath, m_header.pageCount, m_cacheFilePath.c_str());
LOG_DBG("XTC", "Opened file: %s (%u pages, %dx%d)", filepath, m_header.pageCount, m_defaultWidth, m_defaultHeight);
return XtcError::OK;
}
void XtcParser::close() {
closeCacheFile();
if (m_isOpen && m_file.isOpen()) {
m_file.close();
}
closeFile();
m_isOpen = false;
m_l2Valid = false;
m_l2WindowCount = 0;
m_chaptersLoaded = false;
for (auto& entry : m_l1Cache) {
entry.pageIndex = 0xFFFFFFFF;
}
m_chapters.clear();
m_title.clear();
m_author.clear();
m_hasChapters = false;
memset(&m_header, 0, sizeof(m_header));
}
void XtcParser::ensureChaptersLoaded() {
if (m_chaptersLoaded || !m_hasChapters) {
return;
}
// Chapter parsing allocates variable-length strings, so keep it lazy.
const XtcError err = readChapters();
if (err != XtcError::OK) {
LOG_ERR("XTC", "Failed to lazy-load chapters: %s", errorToString(err));
m_hasChapters = false;
m_chapters.clear();
m_chapters.shrink_to_fit();
}
m_chaptersLoaded = true;
}
bool XtcParser::openCacheFile() {
if (m_cacheFile.isOpen()) {
bool XtcParser::ensureFileOpen() {
if (m_file.isOpen()) {
return true;
}
return Storage.openFileForRead("XTC", m_cacheFilePath.c_str(), m_cacheFile);
return Storage.openFileForRead("XTC", m_filepath.c_str(), m_file);
}
void XtcParser::closeCacheFile() {
if (m_cacheFile.isOpen()) {
m_cacheFile.close();
void XtcParser::closeFile() {
if (m_file.isOpen()) {
m_file.close();
}
}
bool XtcParser::getPageInfo(uint32_t pageIndex, PageInfo& info) {
if (pageIndex >= m_header.pageCount) {
return false;
}
// L1 is the hot cache for the most recently used pages.
if (lookupL1(pageIndex, info)) {
LOG_DBG("XTC", "L1 hit: page %u", pageIndex);
return true;
}
// L2 is the sliding window around the reader's current position.
if (lookupL2(pageIndex, info)) {
updateL1(pageIndex, info);
LOG_DBG("XTC", "L2 hit: page %u", pageIndex);
return true;
}
// Fall back to the SD-backed cache file, then refresh L2/L1.
LOG_DBG("XTC", "L3 load: page %u", pageIndex);
loadL2Window(pageIndex);
if (lookupL2(pageIndex, info)) {
updateL1(pageIndex, info);
return true;
}
return false;
}
void XtcParser::prefetchWindow(uint32_t pageIndex) {
if (pageIndex >= m_header.pageCount) {
return;
}
// Avoid reloading the same window when the requested page is already covered.
if (m_l2Valid && pageIndex >= m_l2WindowStart && pageIndex < m_l2WindowStart + m_l2WindowCount) {
return;
}
loadL2Window(pageIndex);
}
bool XtcParser::lookupL1(uint32_t pageIndex, PageInfo& info) {
for (const auto& entry : m_l1Cache) {
if (entry.pageIndex == pageIndex) {
info = entry.info;
return true;
}
}
return false;
}
void XtcParser::updateL1(uint32_t pageIndex, const PageInfo& info) {
for (auto& entry : m_l1Cache) {
if (entry.pageIndex == pageIndex) {
entry.lastAccess = ++m_accessCounter;
return;
}
}
// Replace the least-recently-used entry, or fill the first empty slot.
uint32_t oldestAccess = m_accessCounter;
size_t oldestIndex = 0;
bool foundEmpty = false;
for (size_t i = 0; i < m_l1Cache.size(); i++) {
if (m_l1Cache[i].pageIndex == 0xFFFFFFFF) {
oldestIndex = i;
foundEmpty = true;
break;
}
if (m_l1Cache[i].lastAccess < oldestAccess) {
oldestAccess = m_l1Cache[i].lastAccess;
oldestIndex = i;
}
}
m_l1Cache[oldestIndex].pageIndex = pageIndex;
m_l1Cache[oldestIndex].info = info;
m_l1Cache[oldestIndex].lastAccess = ++m_accessCounter;
}
bool XtcParser::lookupL2(uint32_t pageIndex, PageInfo& info) {
if (!m_l2Valid) {
return false;
}
if (pageIndex >= m_l2WindowStart && pageIndex < m_l2WindowStart + m_l2WindowCount) {
size_t idx = pageIndex - m_l2WindowStart;
info = m_l2Window[idx];
return true;
}
return false;
}
void XtcParser::loadL2Window(uint32_t centerPage) {
// Center the sliding window around the requested page when possible.
uint32_t halfWindow = L2_WINDOW_SIZE / 2;
uint32_t windowStart = (centerPage > halfWindow) ? centerPage - halfWindow : 0;
uint32_t windowEnd = windowStart + L2_WINDOW_SIZE;
if (windowEnd > m_header.pageCount) {
windowEnd = m_header.pageCount;
windowStart = (windowEnd > L2_WINDOW_SIZE) ? windowEnd - L2_WINDOW_SIZE : 0;
}
size_t windowSize = windowEnd - windowStart;
if (windowSize == 0) {
m_l2Valid = false;
m_l2WindowCount = 0;
return;
}
if (!m_cacheFile.isOpen() && !openCacheFile()) {
LOG_ERR("XTC", "Cache file not available");
m_l2Valid = false;
m_l2WindowCount = 0;
return;
}
size_t entryOffset = sizeof(PageTableCacheHeader) + windowStart * sizeof(PageInfo);
if (!m_cacheFile.seek(entryOffset)) {
LOG_ERR("XTC", "Failed to seek in page table cache");
m_l2Valid = false;
m_l2WindowCount = 0;
return;
}
size_t readCount = 0;
for (size_t i = 0; i < windowSize; i++) {
PageInfo info;
if (m_cacheFile.read(reinterpret_cast<uint8_t*>(&info), sizeof(PageInfo)) != sizeof(PageInfo)) {
LOG_ERR("XTC", "Failed to read page info %zu", windowStart + i);
break;
}
m_l2Window[i] = info;
readCount++;
}
m_l2WindowStart = windowStart;
m_l2WindowCount = readCount;
m_l2Valid = (readCount > 0);
LOG_DBG("XTC", "L2 window loaded: [%u, %u] (%zu pages)", windowStart, windowStart + readCount - 1, readCount);
}
bool XtcParser::isPageTableCacheValid() const {
if (!Storage.exists(m_cacheFilePath.c_str())) {
return false;
}
FsFile cacheFile;
if (!Storage.openFileForRead("XTC", m_cacheFilePath.c_str(), cacheFile)) {
return false;
}
uint8_t headerBuf[sizeof(PageTableCacheHeader)];
if (cacheFile.read(headerBuf, sizeof(headerBuf)) != sizeof(headerBuf)) {
cacheFile.close();
return false;
}
PageTableCacheHeader header;
safeDeserializeHeader(headerBuf, header);
if (header.magic != PAGE_TABLE_CACHE_MAGIC || header.version != PAGE_TABLE_CACHE_VERSION) {
cacheFile.close();
return false;
}
// The cache must match both the page count and the original file size.
if (header.pageCount != m_header.pageCount) {
cacheFile.close();
return false;
}
uint32_t expectedSize = sizeof(PageTableCacheHeader) + header.pageCount * sizeof(PageInfo);
if (cacheFile.size() < expectedSize) {
cacheFile.close();
return false;
}
if (header.originalSize > 0) {
FsFile originalFile;
if (Storage.openFileForRead("XTC", m_originalPath.c_str(), originalFile)) {
uint64_t currentSize = originalFile.size();
originalFile.close();
if (currentSize != header.originalSize) {
LOG_INF("XTC", "Cache invalidated: file size changed");
cacheFile.close();
return false;
}
}
}
cacheFile.close();
return true;
}
XtcError XtcParser::buildPageTableCache() {
FsFile originalFile;
if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), originalFile)) {
return XtcError::FILE_NOT_FOUND;
}
size_t lastSlash = m_cacheFilePath.find_last_of('/');
if (lastSlash != std::string::npos) {
std::string cacheDir = m_cacheFilePath.substr(0, lastSlash);
Storage.mkdir(cacheDir.c_str());
}
FsFile cacheFile;
if (!Storage.openFileForWrite("XTC", m_cacheFilePath.c_str(), cacheFile)) {
originalFile.close();
return XtcError::WRITE_ERROR;
}
// Persist a compact PageInfo array so we do not need to hold the full table in RAM.
PageTableCacheHeader header;
header.magic = PAGE_TABLE_CACHE_MAGIC;
header.version = PAGE_TABLE_CACHE_VERSION;
header.pageCount = m_header.pageCount;
header.originalHash = calculateFileHash(m_originalPath.c_str());
header.originalSize = originalFile.size();
header.entrySize = sizeof(PageInfo);
header.reserved = 0;
uint8_t headerBuf[sizeof(PageTableCacheHeader)];
safeSerializeHeader(headerBuf, header);
if (cacheFile.write(headerBuf, sizeof(headerBuf)) != sizeof(headerBuf)) {
cacheFile.close();
originalFile.close();
return XtcError::WRITE_ERROR;
}
if (!seekToOffset(originalFile, m_pageTableOffset)) {
cacheFile.close();
originalFile.close();
return XtcError::READ_ERROR;
}
// Convert the source page table entries into the cached PageInfo layout.
for (uint16_t i = 0; i < m_header.pageCount; i++) {
PageTableEntry entry;
if (originalFile.read(reinterpret_cast<uint8_t*>(&entry), sizeof(PageTableEntry)) != sizeof(PageTableEntry)) {
LOG_ERR("XTC", "Failed to read page table entry %u", i);
cacheFile.close();
originalFile.close();
return XtcError::READ_ERROR;
}
PageInfo info;
info.offset = entry.dataOffset;
info.size = entry.dataSize;
info.width = entry.width;
info.height = entry.height;
info.bitDepth = m_bitDepth;
info.padding = 0;
if (cacheFile.write(reinterpret_cast<const uint8_t*>(&info), sizeof(info)) != sizeof(info)) {
cacheFile.close();
originalFile.close();
return XtcError::WRITE_ERROR;
}
}
cacheFile.close();
originalFile.close();
LOG_INF("XTC", "Page table cache built: %u entries", m_header.pageCount);
return XtcError::OK;
}
uint32_t XtcParser::calculateFileHash(const char* filepath) const {
uint32_t hash = 0;
size_t len = strlen(filepath);
for (size_t i = 0; i < len; i++) {
hash = hash * 31 + static_cast<uint8_t>(filepath[i]);
}
FsFile file;
if (Storage.openFileForRead("XTC", filepath, file)) {
uint64_t size = file.size();
hash ^= static_cast<uint32_t>(size);
hash ^= static_cast<uint32_t>(size >> 32);
file.close();
}
return hash;
}
XtcError XtcParser::readHeader() {
// Read the fixed-size XTC header first.
// Read first 56 bytes of header
size_t bytesRead = m_file.read(reinterpret_cast<uint8_t*>(&m_header), sizeof(XtcHeader));
if (bytesRead != sizeof(XtcHeader)) {
return XtcError::READ_ERROR;
@@ -545,7 +160,7 @@ XtcError XtcParser::readHeader() {
XtcError XtcParser::readTitle() {
constexpr auto titleOffset = 0x38;
if (!m_file.seek(titleOffset)) {
if (!m_file.seek64(titleOffset)) {
return XtcError::READ_ERROR;
}
@@ -560,7 +175,7 @@ XtcError XtcParser::readTitle() {
XtcError XtcParser::readAuthor() {
// Read author as null-terminated UTF-8 string with max length 64, directly following title
constexpr auto authorOffset = 0xB8;
if (!m_file.seek(authorOffset)) {
if (!m_file.seek64(authorOffset)) {
return XtcError::READ_ERROR;
}
@@ -572,20 +187,83 @@ XtcError XtcParser::readAuthor() {
return XtcError::OK;
}
XtcError XtcParser::readChapters() {
m_hasChapters = false;
m_chapters.clear();
m_chapters.shrink_to_fit();
XtcError XtcParser::readFirstPageInfo() {
if (m_header.pageTableOffset == 0) {
LOG_DBG("XTC", "Page table offset is 0, cannot read");
return XtcError::CORRUPTED_HEADER;
}
// Reopen the original file on demand because open() closes it after cache initialization.
if (!m_file.isOpen()) {
if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), m_file)) {
return XtcError::FILE_NOT_FOUND;
}
// Verify the file is large enough to contain the full page table
const uint64_t fileSize = m_file.size64();
const uint64_t pageTableSize = static_cast<uint64_t>(m_header.pageCount) * sizeof(PageTableEntry);
if (m_header.pageTableOffset < sizeof(XtcHeader) || m_header.pageTableOffset > fileSize ||
pageTableSize > fileSize - m_header.pageTableOffset) {
LOG_DBG("XTC", "Page table exceeds file bounds");
return XtcError::CORRUPTED_HEADER;
}
// Read only the first entry to get default page dimensions
// All other entries are read on-demand via readPageTableEntry()
// This avoids allocating pageCount * 16 bytes (e.g. 65KB for 4000+ pages)
PageTableEntry entry;
if (!m_file.seek64(m_header.pageTableOffset)) {
LOG_DBG("XTC", "Failed to seek to page table at %llu", m_header.pageTableOffset);
return XtcError::READ_ERROR;
}
size_t bytesRead = m_file.read(reinterpret_cast<uint8_t*>(&entry), sizeof(PageTableEntry));
if (bytesRead != sizeof(PageTableEntry)) {
LOG_DBG("XTC", "Failed to read first page table entry");
return XtcError::READ_ERROR;
}
m_defaultWidth = entry.width;
m_defaultHeight = entry.height;
LOG_DBG("XTC", "Page table validated: %u pages, default %dx%d", m_header.pageCount, m_defaultWidth, m_defaultHeight);
return XtcError::OK;
}
bool XtcParser::readPageTableEntry(uint32_t pageIndex, PageInfo& info) {
if (pageIndex >= m_header.pageCount) {
return false;
}
if (!ensureFileOpen()) {
LOG_DBG("XTC", "Failed to reopen file for page table read");
return false;
}
// Seek to the specific page table entry on the SD card
const uint64_t entryOffset = m_header.pageTableOffset + static_cast<uint64_t>(pageIndex) * sizeof(PageTableEntry);
if (!m_file.seek64(entryOffset)) {
LOG_DBG("XTC", "Failed to seek to page table entry %lu at %llu", pageIndex, entryOffset);
return false;
}
PageTableEntry entry;
size_t bytesRead = m_file.read(reinterpret_cast<uint8_t*>(&entry), sizeof(PageTableEntry));
if (bytesRead != sizeof(PageTableEntry)) {
LOG_DBG("XTC", "Failed to read page table entry %lu", pageIndex);
return false;
}
info.offset = entry.dataOffset;
info.size = entry.dataSize;
info.width = entry.width;
info.height = entry.height;
info.bitDepth = m_bitDepth;
return true;
}
XtcError XtcParser::readChapters() {
m_chapters.clear();
if (!ensureFileOpen()) {
return XtcError::READ_ERROR;
}
uint8_t hasChaptersFlag = 0;
if (!m_file.seek(0x0B)) {
if (!m_file.seek64(0x0B)) {
return XtcError::READ_ERROR;
}
if (m_file.read(&hasChaptersFlag, sizeof(hasChaptersFlag)) != sizeof(hasChaptersFlag)) {
@@ -597,7 +275,7 @@ XtcError XtcParser::readChapters() {
}
uint64_t chapterOffset = 0;
if (!m_file.seek(0x30)) {
if (!m_file.seek64(0x30)) {
return XtcError::READ_ERROR;
}
if (m_file.read(reinterpret_cast<uint8_t*>(&chapterOffset), sizeof(chapterOffset)) != sizeof(chapterOffset)) {
@@ -608,56 +286,36 @@ XtcError XtcParser::readChapters() {
return XtcError::OK;
}
const uint64_t fileSize = m_file.size();
constexpr size_t chapterSize = 96;
if (chapterOffset < sizeof(XtcHeader) || chapterOffset >= fileSize) {
const uint64_t fileSize = m_file.size64();
if (chapterOffset < sizeof(XtcHeader) || chapterOffset >= fileSize || chapterOffset + 96 > fileSize) {
return XtcError::OK;
}
if (fileSize - chapterOffset < chapterSize) {
return XtcError::OK;
}
uint64_t maxOffset = 0;
if (m_header.pageTableOffset > chapterOffset) {
// Clamp maxOffset to fileSize so bogus header values can't inflate chapterCount
uint64_t maxOffset = fileSize;
if (m_header.pageTableOffset > chapterOffset && m_header.pageTableOffset <= fileSize) {
maxOffset = m_header.pageTableOffset;
} else if (m_header.dataOffset > chapterOffset) {
} else if (m_header.dataOffset > chapterOffset && m_header.dataOffset <= fileSize) {
maxOffset = m_header.dataOffset;
} else {
maxOffset = fileSize;
}
if (maxOffset <= chapterOffset) {
return XtcError::OK;
}
constexpr size_t chapterSize = 96;
const uint64_t available = maxOffset - chapterOffset;
const uint64_t chapterCount64 = available / chapterSize;
if (chapterCount64 == 0) {
return XtcError::OK;
}
if (chapterCount64 > MAX_CHAPTERS || chapterCount64 > std::numeric_limits<size_t>::max()) {
LOG_ERR("XTC", "Chapter table too large: available=%llu chapterCount=%llu",
static_cast<unsigned long long>(available), static_cast<unsigned long long>(chapterCount64));
return XtcError::CORRUPTED_HEADER;
}
const size_t chapterCount = static_cast<size_t>(chapterCount64);
const size_t chapterCount = static_cast<size_t>(available / chapterSize);
if (chapterCount == 0) {
return XtcError::OK;
}
const size_t freeHeapBefore = ESP.getFreeHeap();
const size_t maxAllocBefore = ESP.getMaxAllocHeap();
if (!seekToOffset(m_file, chapterOffset)) {
if (!m_file.seek64(chapterOffset)) {
return XtcError::READ_ERROR;
}
std::vector<uint8_t> chapterBuf(chapterSize);
m_chapters.reserve(chapterCount);
std::vector<uint8_t> chapterBuf(chapterSize);
for (size_t i = 0; i < chapterCount; i++) {
if (m_file.read(chapterBuf.data(), chapterSize) != chapterSize) {
return XtcError::READ_ERROR;
@@ -701,29 +359,29 @@ XtcError XtcParser::readChapters() {
m_chapters.push_back(std::move(chapter));
}
m_chapters.shrink_to_fit();
m_hasChapters = !m_chapters.empty();
size_t chapterNameBytes = 0;
for (const auto& chapter : m_chapters) {
chapterNameBytes += chapter.name.capacity() + 1;
}
const size_t chapterVectorBytes = m_chapters.capacity() * sizeof(ChapterInfo);
const size_t totalChapterBytes = chapterVectorBytes + chapterNameBytes;
const size_t freeHeapAfter = ESP.getFreeHeap();
const size_t maxAllocAfter = ESP.getMaxAllocHeap();
const int heapDelta = static_cast<int>(freeHeapBefore) - static_cast<int>(freeHeapAfter);
const int maxAllocDelta = static_cast<int>(maxAllocBefore) - static_cast<int>(maxAllocAfter);
LOG_INF("XTC", "Chapter metadata: count=%u, vector~=%zu, names~=%zu, total~=%zu, heapDelta=%d, maxAllocDelta=%d",
static_cast<unsigned int>(m_chapters.size()), chapterVectorBytes, chapterNameBytes, totalChapterBytes,
heapDelta, maxAllocDelta);
LOG_DBG("XTC", "Chapters: %u", static_cast<unsigned int>(m_chapters.size()));
return XtcError::OK;
}
const std::vector<ChapterInfo>& XtcParser::getChapters() {
ensureChaptersLoaded();
// Lazy load chapters on first access
if (!m_chaptersLoaded && m_hasChapters) {
const XtcError err = readChapters();
if (err != XtcError::OK) {
LOG_ERR("XTC", "Failed to lazy-load chapters: %s", errorToString(err));
m_hasChapters = false;
m_chapters.clear();
}
m_chaptersLoaded = true;
// Close file after chapter read to free buffers for rendering
closeFile();
}
return m_chapters;
}
bool XtcParser::getPageInfo(uint32_t pageIndex, PageInfo& info) { return readPageTableEntry(pageIndex, info); }
size_t XtcParser::loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSize) {
if (!m_isOpen) {
m_lastError = XtcError::FILE_NOT_FOUND;
@@ -735,23 +393,20 @@ size_t XtcParser::loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSiz
return 0;
}
// Resolve the page location through the cache hierarchy before touching the data file.
PageInfo info;
if (!getPageInfo(pageIndex, info)) {
PageInfo page;
if (!readPageTableEntry(pageIndex, page)) {
m_lastError = XtcError::READ_ERROR;
return 0;
}
// Reopen the source file lazily because normal parser open() does not keep it pinned.
if (!m_file.isOpen()) {
if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), m_file)) {
m_lastError = XtcError::FILE_NOT_FOUND;
return 0;
}
if (!ensureFileOpen()) {
m_lastError = XtcError::FILE_NOT_FOUND;
return 0;
}
if (!seekToOffset(m_file, info.offset)) {
LOG_DBG("XTC", "Failed to seek to page %u at offset %llu", pageIndex, static_cast<unsigned long long>(info.offset));
// Seek to page data
if (!m_file.seek64(page.offset)) {
LOG_DBG("XTC", "Failed to seek to page %u at offset %lu", pageIndex, page.offset);
m_lastError = XtcError::READ_ERROR;
return 0;
}
@@ -785,14 +440,14 @@ size_t XtcParser::loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSiz
bitmapSize = ((pageHeader.width + 7) / 8) * pageHeader.height;
}
// The caller owns the buffer, so fail early if it is too small.
// Check buffer size
if (bufferSize < bitmapSize) {
LOG_DBG("XTC", "Buffer too small: need %u, have %u", bitmapSize, bufferSize);
m_lastError = XtcError::MEMORY_ERROR;
return 0;
}
// Read the bitmap payload into the caller-provided buffer.
// Read bitmap data
size_t bytesRead = m_file.read(buffer, bitmapSize);
if (bytesRead != bitmapSize) {
LOG_DBG("XTC", "Page read error: expected %u, got %u", bitmapSize, bytesRead);
@@ -815,25 +470,21 @@ XtcError XtcParser::loadPageStreaming(uint32_t pageIndex,
return XtcError::PAGE_OUT_OF_RANGE;
}
// Streaming uses the same cache lookup path but reads the payload in chunks.
PageInfo info;
if (!getPageInfo(pageIndex, info)) {
PageInfo page;
if (!readPageTableEntry(pageIndex, page)) {
return XtcError::READ_ERROR;
}
// Reopen the source file on demand for streaming reads as well.
if (!m_file.isOpen()) {
if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), m_file)) {
return XtcError::FILE_NOT_FOUND;
}
if (!ensureFileOpen()) {
return XtcError::FILE_NOT_FOUND;
}
if (!seekToOffset(m_file, info.offset)) {
LOG_DBG("XTC", "Failed to seek to page %u at offset %llu", pageIndex, static_cast<unsigned long long>(info.offset));
// Seek to page data
if (!m_file.seek64(page.offset)) {
return XtcError::READ_ERROR;
}
// Read and validate the page header before yielding any bitmap bytes.
// Read and skip page header (XTG for 1-bit, XTH for 2-bit)
XtgPageHeader pageHeader;
size_t headerRead = m_file.read(reinterpret_cast<uint8_t*>(&pageHeader), sizeof(XtgPageHeader));
const uint32_t expectedMagic = (m_bitDepth == 2) ? XTH_MAGIC : XTG_MAGIC;
@@ -844,7 +495,6 @@ XtcError XtcParser::loadPageStreaming(uint32_t pageIndex,
// Calculate bitmap size based on bit depth
// XTG (1-bit): Row-major, ((width+7)/8) * height bytes
// XTH (2-bit): Two bit planes, ((width * height + 7) / 8) * 2 bytes
// Match the bitmap sizing rules used by the non-streaming path.
size_t bitmapSize;
if (m_bitDepth == 2) {
bitmapSize = ((static_cast<size_t>(pageHeader.width) * pageHeader.height + 7) / 8) * 2;
@@ -852,7 +502,7 @@ XtcError XtcParser::loadPageStreaming(uint32_t pageIndex,
bitmapSize = ((pageHeader.width + 7) / 8) * pageHeader.height;
}
// Feed the bitmap to the callback in bounded chunks to keep peak memory low.
// Read in chunks
std::vector<uint8_t> chunk(chunkSize);
size_t totalRead = 0;
@@ -888,4 +538,4 @@ bool XtcParser::isValidXtcFile(const char* filepath) {
return (magic == XTC_MAGIC || magic == XTCH_MAGIC);
}
} // namespace xtc
} // namespace xtc
lib/Xtc/Xtc/XtcParser.h
+32 -49
View File
@@ -9,8 +9,6 @@
#include <HalStorage.h>
#include <array>
#include <cstring>
#include <functional>
#include <memory>
#include <string>
@@ -25,6 +23,9 @@ namespace xtc {
*
* Reads XTC files from SD card and extracts page data.
* Designed for ESP32-C3's limited RAM (~380KB) using streaming.
*
* The source file is kept closed between reads to free heap for rendering.
* It is reopened on-demand for page table lookups and bitmap data reads.
*/
class XtcParser {
public:
@@ -32,7 +33,7 @@ class XtcParser {
~XtcParser();
// File open/close
XtcError open(const char* filepath, const char* cacheDir);
XtcError open(const char* filepath);
void close();
bool isOpen() const { return m_isOpen; }
@@ -43,14 +44,28 @@ class XtcParser {
uint16_t getHeight() const { return m_defaultHeight; }
uint8_t getBitDepth() const { return m_bitDepth; } // 1 = XTC/XTG, 2 = XTCH/XTH
// Page information - three-tier cache interface
// Page information
bool getPageInfo(uint32_t pageIndex, PageInfo& info);
// Preload window around specified page (optimize sequential page turns)
void prefetchWindow(uint32_t pageIndex);
// Load page bitmap (unchanged)
/**
* Load page bitmap (raw 1-bit data, skipping XTG header)
*
* @param pageIndex Page index (0-based)
* @param buffer Output buffer (caller allocated)
* @param bufferSize Buffer size
* @return Number of bytes read on success, 0 on failure
*/
size_t loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSize);
/**
* Streaming page load
* Memory-efficient method that reads page data in chunks.
*
* @param pageIndex Page index
* @param callback Callback function to receive data chunks
* @param chunkSize Chunk size (default: 1024 bytes)
* @return Error code
*/
XtcError loadPageStreaming(uint32_t pageIndex,
std::function<void(const uint8_t* data, size_t size, size_t offset)> callback,
size_t chunkSize = 1024);
@@ -70,62 +85,30 @@ class XtcParser {
private:
FsFile m_file;
FsFile m_cacheFile;
std::string m_filepath;
bool m_isOpen;
XtcHeader m_header;
std::string m_cacheDir;
std::string m_cacheFilePath;
std::string m_originalPath;
std::vector<ChapterInfo> m_chapters;
std::string m_title;
std::string m_author;
uint16_t m_defaultWidth;
uint16_t m_defaultHeight;
uint8_t m_bitDepth;
uint8_t m_bitDepth; // 1 = XTC/XTG (1-bit), 2 = XTCH/XTH (2-bit)
bool m_hasChapters;
bool m_chaptersLoaded = false;
bool m_chaptersLoaded;
XtcError m_lastError;
uint32_t m_accessCounter = 0;
// L1: Hot cache (fixed 4 entries)
std::array<L1CacheEntry, L1_CACHE_SIZE> m_l1Cache;
// L2: Sliding window (fixed size array)
std::array<PageInfo, L2_WINDOW_SIZE> m_l2Window;
uint32_t m_l2WindowStart = 0;
size_t m_l2WindowCount = 0;
bool m_l2Valid = false;
// Chapters (usually few, keep in memory)
std::vector<ChapterInfo> m_chapters;
// Original Page Table offset (for rebuilding cache)
uint64_t m_pageTableOffset = 0;
// Internal helper functions
XtcError readHeader();
XtcError readFirstPageInfo();
XtcError readTitle();
XtcError readAuthor();
XtcError readChapters();
void ensureChaptersLoaded();
bool readPageTableEntry(uint32_t pageIndex, PageInfo& info);
// L3 cache management
bool isPageTableCacheValid() const;
XtcError buildPageTableCache();
bool openCacheFile();
void closeCacheFile();
// L1/L2 cache operations
bool lookupL1(uint32_t pageIndex, PageInfo& info);
void updateL1(uint32_t pageIndex, const PageInfo& info);
bool lookupL2(uint32_t pageIndex, PageInfo& info);
void loadL2Window(uint32_t centerPage);
// Safe deserialization (alignment-safe for ESP32-C3)
static void safeDeserializeHeader(const uint8_t* buf, PageTableCacheHeader& header);
static void safeSerializeHeader(uint8_t* buf, const PageTableCacheHeader& header);
// Utility functions
uint32_t calculateFileHash(const char* filepath) const;
// File handle management — reopen on demand, close after use
bool ensureFileOpen();
void closeFile();
};
} // namespace xtc
lib/Xtc/Xtc/XtcTypes.h
-27
View File
@@ -102,33 +102,6 @@ struct ChapterInfo {
uint16_t endPage;
};
// Cache configuration
constexpr size_t L1_CACHE_SIZE = 4; // L1 cache entries
constexpr size_t L2_WINDOW_SIZE = 100; // L2 window size (reduced for 2-bit memory)
constexpr uint32_t PAGE_TABLE_CACHE_VERSION = 1; // Cache file version
// Cache magic number
constexpr uint32_t PAGE_TABLE_CACHE_MAGIC = 0x50435458; // "XTCP"
// Cache file header - NOTE: Do NOT read directly from file buffer!
// Use safeDeserializeHeader() function for alignment-safe access.
struct PageTableCacheHeader {
uint32_t magic; // 'XTCP' = 0x50435458
uint32_t version; // Cache version
uint32_t pageCount; // Total pages
uint32_t originalHash; // Original file hash (for validation)
uint64_t originalSize; // Original file size (for validation)
uint32_t entrySize; // PageInfo size (16)
uint32_t reserved; // Reserved
};
// L1 cache entry
struct L1CacheEntry {
uint32_t pageIndex = 0xFFFFFFFF; // 0xFFFFFFFF = invalid
PageInfo info{};
uint32_t lastAccess = 0; // Timestamp for LRU
};
// Error codes
enum class XtcError {
OK = 0,