diff --git a/lib/Xtc/Xtc.cpp b/lib/Xtc/Xtc.cpp index 6dd3886b..a1f8f8a6 100644 --- a/lib/Xtc/Xtc.cpp +++ b/lib/Xtc/Xtc.cpp @@ -14,14 +14,11 @@ bool Xtc::load() { LOG_DBG("XTC", "Loading XTC: %s", filepath.c_str()); - // Ensure the per-book cache exists before the parser tries to create page_table.bin. - setupCacheDir(); - // Initialize parser parser.reset(new xtc::XtcParser()); - // Open XTC file and initialize its cache-backed page table - xtc::XtcError err = parser->open(filepath.c_str(), cachePath.c_str()); + // Open XTC file + xtc::XtcError err = parser->open(filepath.c_str()); if (err != xtc::XtcError::OK) { LOG_ERR("XTC", "Failed to load: %s", xtc::errorToString(err)); parser.reset(); @@ -106,7 +103,7 @@ bool Xtc::hasChapters() const { return parser->hasChapters(); } -const std::vector& Xtc::getChapters() const { +const std::vector& Xtc::getChapters() { static const std::vector kEmpty; if (!loaded || !parser) { return kEmpty; @@ -202,7 +199,6 @@ bool Xtc::generateCoverBmp() const { uint8_t* rowBuffer = static_cast(malloc(dstRowSize)); if (!rowBuffer) { free(pageBuffer); - coverBmp.close(); return false; } @@ -258,7 +254,6 @@ bool Xtc::generateCoverBmp() const { } } - coverBmp.close(); free(pageBuffer); LOG_DBG("XTC", "Generated cover BMP: %s", getCoverBmpPath().c_str()); @@ -319,9 +314,7 @@ bool Xtc::generateThumbBmp(int height) const { size_t bytesRead = src.read(buffer, sizeof(buffer)); dst.write(buffer, bytesRead); } - dst.close(); } - src.close(); } LOG_DBG("XTC", "Copied cover to thumb (no scaling needed)"); return Storage.exists(getThumbBmpPath(height).c_str()); @@ -375,7 +368,6 @@ bool Xtc::generateThumbBmp(int height) const { uint8_t* rowBuffer = static_cast(malloc(rowSize)); if (!rowBuffer) { free(pageBuffer); - thumbBmp.close(); return false; } @@ -482,7 +474,6 @@ bool Xtc::generateThumbBmp(int height) const { } free(rowBuffer); - thumbBmp.close(); free(pageBuffer); LOG_DBG("XTC", "Generated thumb BMP (%dx%d): %s", thumbWidth, thumbHeight, getThumbBmpPath(height).c_str()); @@ -545,11 +536,4 @@ xtc::XtcError Xtc::getLastError() const { return xtc::XtcError::FILE_NOT_FOUND; } return parser->getLastError(); -} - -void Xtc::prefetchPages(uint32_t pageIndex) const { - if (!loaded || !parser) { - return; - } - parser->prefetchWindow(pageIndex); } \ No newline at end of file diff --git a/lib/Xtc/Xtc.h b/lib/Xtc/Xtc.h index 1c79416f..93f911fc 100644 --- a/lib/Xtc/Xtc.h +++ b/lib/Xtc/Xtc.h @@ -50,11 +50,6 @@ class Xtc { */ void setupCacheDir() const; - /** - * Preload window around specified page (for page turn optimization) - */ - void prefetchPages(uint32_t pageIndex) const; - // Path accessors const std::string& getCachePath() const { return cachePath; } const std::string& getPath() const { return filepath; } @@ -63,7 +58,7 @@ class Xtc { std::string getTitle() const; std::string getAuthor() const; bool hasChapters() const; - const std::vector& getChapters() const; + const std::vector& getChapters(); // Cover image support (for sleep screen) std::string getCoverBmpPath() const; @@ -107,4 +102,4 @@ class Xtc { // Error information xtc::XtcError getLastError() const; -}; +}; \ No newline at end of file diff --git a/lib/Xtc/Xtc/XtcParser.cpp b/lib/Xtc/Xtc/XtcParser.cpp index 083391eb..64a4e27c 100644 --- a/lib/Xtc/Xtc/XtcParser.cpp +++ b/lib/Xtc/Xtc/XtcParser.cpp @@ -10,80 +10,33 @@ #include #include #include -#include #include -#include namespace xtc { -namespace { - -constexpr size_t MAX_CHAPTERS = 4096; - -bool canSeekToOffset(const uint64_t offset) { - return offset <= static_cast(std::numeric_limits::max()); -} - -bool seekToOffset(FsFile& file, const uint64_t offset) { - if (!canSeekToOffset(offset)) { - return false; - } - return file.seek(static_cast(offset)); -} - -} // namespace - -void XtcParser::safeDeserializeHeader(const uint8_t* buf, PageTableCacheHeader& header) { - memcpy(&header.magic, buf + 0, 4); - memcpy(&header.version, buf + 4, 4); - memcpy(&header.pageCount, buf + 8, 4); - memcpy(&header.originalHash, buf + 12, 4); - memcpy(&header.originalSize, buf + 16, 8); - memcpy(&header.entrySize, buf + 24, 4); - memcpy(&header.reserved, buf + 28, 4); -} - -void XtcParser::safeSerializeHeader(uint8_t* buf, const PageTableCacheHeader& header) { - memcpy(buf + 0, &header.magic, 4); - memcpy(buf + 4, &header.version, 4); - memcpy(buf + 8, &header.pageCount, 4); - memcpy(buf + 12, &header.originalHash, 4); - memcpy(buf + 16, &header.originalSize, 8); - memcpy(buf + 24, &header.entrySize, 4); - memcpy(buf + 28, &header.reserved, 4); -} - XtcParser::XtcParser() : m_isOpen(false), m_defaultWidth(DISPLAY_WIDTH), m_defaultHeight(DISPLAY_HEIGHT), m_bitDepth(1), m_hasChapters(false), + m_chaptersLoaded(false), m_lastError(XtcError::OK) { memset(&m_header, 0, sizeof(m_header)); - - for (auto& entry : m_l1Cache) { - entry.pageIndex = 0xFFFFFFFF; - entry.lastAccess = 0; - } } XtcParser::~XtcParser() { close(); } -XtcError XtcParser::open(const char* filepath, const char* cacheDir) { - // Close any previous file state before reopening +XtcError XtcParser::open(const char* filepath) { + // Close if already open if (m_isOpen) { close(); } - m_originalPath = filepath; - m_cacheDir = cacheDir; + m_filepath = filepath; - uint32_t fileHash = calculateFileHash(filepath); - m_cacheFilePath = std::string(cacheDir) + "/xtc_" + std::to_string(fileHash) + "/page_table.bin"; - - // Open the original XTC file just long enough to read metadata and validate the header. + // Open file if (!Storage.openFileForRead("XTC", filepath, m_file)) { m_lastError = XtcError::FILE_NOT_FOUND; return m_lastError; @@ -93,420 +46,82 @@ XtcError XtcParser::open(const char* filepath, const char* cacheDir) { m_lastError = readHeader(); if (m_lastError != XtcError::OK) { LOG_DBG("XTC", "Failed to read header: %s", errorToString(m_lastError)); + // Explicit close() required: member variable persists beyond function scope m_file.close(); return m_lastError; } - if (m_header.pageCount == 0) { - LOG_ERR("XTC", "File has no pages"); - m_file.close(); - m_lastError = XtcError::CORRUPTED_HEADER; - return m_lastError; - } - - // Metadata strings are small, so keep them in memory even when the page table is moved to cache. + // Read title & author if available if (m_header.hasMetadata) { - readTitle(); - readAuthor(); - m_title.shrink_to_fit(); - m_author.shrink_to_fit(); - LOG_INF("XTC", "Metadata strings: titleLen=%u cap=%u, authorLen=%u cap=%u", - static_cast(m_title.size()), static_cast(m_title.capacity()), - static_cast(m_author.size()), static_cast(m_author.capacity())); - } - - // Defer chapter parsing until the reader actually needs the table of contents. - m_pageTableOffset = m_header.pageTableOffset; - m_hasChapters = (m_header.hasChapters == 1) && (m_header.chapterOffset != 0); - LOG_INF("XTC", "Chapter metadata deferred: available=%s", m_hasChapters ? "yes" : "no"); - - m_file.close(); - - // Build or reuse the on-disk page table cache before marking the parser open. - if (!isPageTableCacheValid()) { - LOG_INF("XTC", "Building page table cache for %u pages", m_header.pageCount); - m_lastError = buildPageTableCache(); + m_lastError = readTitle(); if (m_lastError != XtcError::OK) { - LOG_ERR("XTC", "Failed to build page table cache"); + LOG_DBG("XTC", "Failed to read title: %s", errorToString(m_lastError)); + // Explicit close() required: member variable persists beyond function scope + m_file.close(); return m_lastError; } - const size_t heapBefore = ESP.getMaxAllocHeap(); - LOG_DBG("XTC", "Cache built, heap before defrag: free=%zu, maxAlloc=%zu", ESP.getFreeHeap(), heapBefore); - - // Defragment heap: small delay allows heap coalescing after file handles are closed - // This typically improves MaxAlloc by 10-20KB, enabling 96KB page buffer for grayscale - LOG_DBG("XTC", "Defragmenting heap (waiting 50ms)..."); - vTaskDelay(pdMS_TO_TICKS(50)); - - const size_t heapAfter = ESP.getMaxAllocHeap(); - const size_t heapGain = heapAfter > heapBefore ? (heapAfter - heapBefore) : 0; - if (heapGain > 0) { - LOG_INF("XTC", "Heap defragmented: +%zu bytes contiguous (now %zu)", heapGain, heapAfter); - } else { - LOG_DBG("XTC", "Heap after defrag: free=%zu, maxAlloc=%zu", ESP.getFreeHeap(), heapAfter); + m_lastError = readAuthor(); + if (m_lastError != XtcError::OK) { + LOG_DBG("XTC", "Failed to read author: %s", errorToString(m_lastError)); + // Explicit close() required: member variable persists beyond function scope + m_file.close(); + return m_lastError; } + // Trim excess capacity from metadata strings + m_title.shrink_to_fit(); + m_author.shrink_to_fit(); } - if (!openCacheFile()) { - LOG_ERR("XTC", "Failed to open cache file"); - m_lastError = XtcError::FILE_NOT_FOUND; + // Read first page info for default dimensions (no bulk page table allocation) + m_lastError = readFirstPageInfo(); + if (m_lastError != XtcError::OK) { + LOG_DBG("XTC", "Failed to read first page info: %s", errorToString(m_lastError)); + // Explicit close() required: member variable persists beyond function scope + m_file.close(); return m_lastError; } - // Prime the sliding L2 window with the first chunk of page metadata. - loadL2Window(0); + // Defer chapter parsing until actually needed (lazy load). + // Chapter strings can use significant heap; keeping them out of memory + // during rendering leaves more room for the page bitmap buffer. + m_hasChapters = (m_header.hasChapters == 1); + m_chaptersLoaded = false; - LOG_DBG("XTC", "File opened, heap: free=%zu, maxAlloc=%zu", ESP.getFreeHeap(), ESP.getMaxAllocHeap()); + // Close the source file to free its internal SdFat buffers. + // It will be reopened on-demand for page table lookups and bitmap reads. + m_file.close(); m_isOpen = true; - LOG_DBG("XTC", "Opened file: %s (%u pages, cache: %s)", filepath, m_header.pageCount, m_cacheFilePath.c_str()); + LOG_DBG("XTC", "Opened file: %s (%u pages, %dx%d)", filepath, m_header.pageCount, m_defaultWidth, m_defaultHeight); return XtcError::OK; } void XtcParser::close() { - closeCacheFile(); - - if (m_isOpen && m_file.isOpen()) { - m_file.close(); - } + closeFile(); m_isOpen = false; - m_l2Valid = false; - m_l2WindowCount = 0; m_chaptersLoaded = false; - - for (auto& entry : m_l1Cache) { - entry.pageIndex = 0xFFFFFFFF; - } m_chapters.clear(); m_title.clear(); m_author.clear(); + m_hasChapters = false; memset(&m_header, 0, sizeof(m_header)); } -void XtcParser::ensureChaptersLoaded() { - if (m_chaptersLoaded || !m_hasChapters) { - return; - } - - // Chapter parsing allocates variable-length strings, so keep it lazy. - const XtcError err = readChapters(); - if (err != XtcError::OK) { - LOG_ERR("XTC", "Failed to lazy-load chapters: %s", errorToString(err)); - m_hasChapters = false; - m_chapters.clear(); - m_chapters.shrink_to_fit(); - } - m_chaptersLoaded = true; -} - -bool XtcParser::openCacheFile() { - if (m_cacheFile.isOpen()) { +bool XtcParser::ensureFileOpen() { + if (m_file.isOpen()) { return true; } - return Storage.openFileForRead("XTC", m_cacheFilePath.c_str(), m_cacheFile); + return Storage.openFileForRead("XTC", m_filepath.c_str(), m_file); } -void XtcParser::closeCacheFile() { - if (m_cacheFile.isOpen()) { - m_cacheFile.close(); +void XtcParser::closeFile() { + if (m_file.isOpen()) { + m_file.close(); } } -bool XtcParser::getPageInfo(uint32_t pageIndex, PageInfo& info) { - if (pageIndex >= m_header.pageCount) { - return false; - } - - // L1 is the hot cache for the most recently used pages. - if (lookupL1(pageIndex, info)) { - LOG_DBG("XTC", "L1 hit: page %u", pageIndex); - return true; - } - - // L2 is the sliding window around the reader's current position. - if (lookupL2(pageIndex, info)) { - updateL1(pageIndex, info); - LOG_DBG("XTC", "L2 hit: page %u", pageIndex); - return true; - } - - // Fall back to the SD-backed cache file, then refresh L2/L1. - LOG_DBG("XTC", "L3 load: page %u", pageIndex); - loadL2Window(pageIndex); - - if (lookupL2(pageIndex, info)) { - updateL1(pageIndex, info); - return true; - } - - return false; -} - -void XtcParser::prefetchWindow(uint32_t pageIndex) { - if (pageIndex >= m_header.pageCount) { - return; - } - - // Avoid reloading the same window when the requested page is already covered. - if (m_l2Valid && pageIndex >= m_l2WindowStart && pageIndex < m_l2WindowStart + m_l2WindowCount) { - return; - } - - loadL2Window(pageIndex); -} - -bool XtcParser::lookupL1(uint32_t pageIndex, PageInfo& info) { - for (const auto& entry : m_l1Cache) { - if (entry.pageIndex == pageIndex) { - info = entry.info; - return true; - } - } - return false; -} - -void XtcParser::updateL1(uint32_t pageIndex, const PageInfo& info) { - for (auto& entry : m_l1Cache) { - if (entry.pageIndex == pageIndex) { - entry.lastAccess = ++m_accessCounter; - return; - } - } - - // Replace the least-recently-used entry, or fill the first empty slot. - uint32_t oldestAccess = m_accessCounter; - size_t oldestIndex = 0; - bool foundEmpty = false; - - for (size_t i = 0; i < m_l1Cache.size(); i++) { - if (m_l1Cache[i].pageIndex == 0xFFFFFFFF) { - oldestIndex = i; - foundEmpty = true; - break; - } - if (m_l1Cache[i].lastAccess < oldestAccess) { - oldestAccess = m_l1Cache[i].lastAccess; - oldestIndex = i; - } - } - - m_l1Cache[oldestIndex].pageIndex = pageIndex; - m_l1Cache[oldestIndex].info = info; - m_l1Cache[oldestIndex].lastAccess = ++m_accessCounter; -} - -bool XtcParser::lookupL2(uint32_t pageIndex, PageInfo& info) { - if (!m_l2Valid) { - return false; - } - - if (pageIndex >= m_l2WindowStart && pageIndex < m_l2WindowStart + m_l2WindowCount) { - size_t idx = pageIndex - m_l2WindowStart; - info = m_l2Window[idx]; - return true; - } - return false; -} - -void XtcParser::loadL2Window(uint32_t centerPage) { - // Center the sliding window around the requested page when possible. - uint32_t halfWindow = L2_WINDOW_SIZE / 2; - uint32_t windowStart = (centerPage > halfWindow) ? centerPage - halfWindow : 0; - uint32_t windowEnd = windowStart + L2_WINDOW_SIZE; - - if (windowEnd > m_header.pageCount) { - windowEnd = m_header.pageCount; - windowStart = (windowEnd > L2_WINDOW_SIZE) ? windowEnd - L2_WINDOW_SIZE : 0; - } - - size_t windowSize = windowEnd - windowStart; - if (windowSize == 0) { - m_l2Valid = false; - m_l2WindowCount = 0; - return; - } - - if (!m_cacheFile.isOpen() && !openCacheFile()) { - LOG_ERR("XTC", "Cache file not available"); - m_l2Valid = false; - m_l2WindowCount = 0; - return; - } - - size_t entryOffset = sizeof(PageTableCacheHeader) + windowStart * sizeof(PageInfo); - if (!m_cacheFile.seek(entryOffset)) { - LOG_ERR("XTC", "Failed to seek in page table cache"); - m_l2Valid = false; - m_l2WindowCount = 0; - return; - } - - size_t readCount = 0; - for (size_t i = 0; i < windowSize; i++) { - PageInfo info; - if (m_cacheFile.read(reinterpret_cast(&info), sizeof(PageInfo)) != sizeof(PageInfo)) { - LOG_ERR("XTC", "Failed to read page info %zu", windowStart + i); - break; - } - m_l2Window[i] = info; - readCount++; - } - - m_l2WindowStart = windowStart; - m_l2WindowCount = readCount; - m_l2Valid = (readCount > 0); - - LOG_DBG("XTC", "L2 window loaded: [%u, %u] (%zu pages)", windowStart, windowStart + readCount - 1, readCount); -} - -bool XtcParser::isPageTableCacheValid() const { - if (!Storage.exists(m_cacheFilePath.c_str())) { - return false; - } - - FsFile cacheFile; - if (!Storage.openFileForRead("XTC", m_cacheFilePath.c_str(), cacheFile)) { - return false; - } - - uint8_t headerBuf[sizeof(PageTableCacheHeader)]; - if (cacheFile.read(headerBuf, sizeof(headerBuf)) != sizeof(headerBuf)) { - cacheFile.close(); - return false; - } - - PageTableCacheHeader header; - safeDeserializeHeader(headerBuf, header); - - if (header.magic != PAGE_TABLE_CACHE_MAGIC || header.version != PAGE_TABLE_CACHE_VERSION) { - cacheFile.close(); - return false; - } - - // The cache must match both the page count and the original file size. - if (header.pageCount != m_header.pageCount) { - cacheFile.close(); - return false; - } - - uint32_t expectedSize = sizeof(PageTableCacheHeader) + header.pageCount * sizeof(PageInfo); - if (cacheFile.size() < expectedSize) { - cacheFile.close(); - return false; - } - - if (header.originalSize > 0) { - FsFile originalFile; - if (Storage.openFileForRead("XTC", m_originalPath.c_str(), originalFile)) { - uint64_t currentSize = originalFile.size(); - originalFile.close(); - if (currentSize != header.originalSize) { - LOG_INF("XTC", "Cache invalidated: file size changed"); - cacheFile.close(); - return false; - } - } - } - - cacheFile.close(); - return true; -} - -XtcError XtcParser::buildPageTableCache() { - FsFile originalFile; - if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), originalFile)) { - return XtcError::FILE_NOT_FOUND; - } - - size_t lastSlash = m_cacheFilePath.find_last_of('/'); - if (lastSlash != std::string::npos) { - std::string cacheDir = m_cacheFilePath.substr(0, lastSlash); - Storage.mkdir(cacheDir.c_str()); - } - - FsFile cacheFile; - if (!Storage.openFileForWrite("XTC", m_cacheFilePath.c_str(), cacheFile)) { - originalFile.close(); - return XtcError::WRITE_ERROR; - } - - // Persist a compact PageInfo array so we do not need to hold the full table in RAM. - PageTableCacheHeader header; - header.magic = PAGE_TABLE_CACHE_MAGIC; - header.version = PAGE_TABLE_CACHE_VERSION; - header.pageCount = m_header.pageCount; - header.originalHash = calculateFileHash(m_originalPath.c_str()); - header.originalSize = originalFile.size(); - header.entrySize = sizeof(PageInfo); - header.reserved = 0; - - uint8_t headerBuf[sizeof(PageTableCacheHeader)]; - safeSerializeHeader(headerBuf, header); - if (cacheFile.write(headerBuf, sizeof(headerBuf)) != sizeof(headerBuf)) { - cacheFile.close(); - originalFile.close(); - return XtcError::WRITE_ERROR; - } - - if (!seekToOffset(originalFile, m_pageTableOffset)) { - cacheFile.close(); - originalFile.close(); - return XtcError::READ_ERROR; - } - - // Convert the source page table entries into the cached PageInfo layout. - for (uint16_t i = 0; i < m_header.pageCount; i++) { - PageTableEntry entry; - if (originalFile.read(reinterpret_cast(&entry), sizeof(PageTableEntry)) != sizeof(PageTableEntry)) { - LOG_ERR("XTC", "Failed to read page table entry %u", i); - cacheFile.close(); - originalFile.close(); - return XtcError::READ_ERROR; - } - - PageInfo info; - info.offset = entry.dataOffset; - info.size = entry.dataSize; - info.width = entry.width; - info.height = entry.height; - info.bitDepth = m_bitDepth; - info.padding = 0; - - if (cacheFile.write(reinterpret_cast(&info), sizeof(info)) != sizeof(info)) { - cacheFile.close(); - originalFile.close(); - return XtcError::WRITE_ERROR; - } - } - - cacheFile.close(); - originalFile.close(); - - LOG_INF("XTC", "Page table cache built: %u entries", m_header.pageCount); - return XtcError::OK; -} - -uint32_t XtcParser::calculateFileHash(const char* filepath) const { - uint32_t hash = 0; - size_t len = strlen(filepath); - - for (size_t i = 0; i < len; i++) { - hash = hash * 31 + static_cast(filepath[i]); - } - - FsFile file; - if (Storage.openFileForRead("XTC", filepath, file)) { - uint64_t size = file.size(); - hash ^= static_cast(size); - hash ^= static_cast(size >> 32); - file.close(); - } - - return hash; -} - XtcError XtcParser::readHeader() { - // Read the fixed-size XTC header first. + // Read first 56 bytes of header size_t bytesRead = m_file.read(reinterpret_cast(&m_header), sizeof(XtcHeader)); if (bytesRead != sizeof(XtcHeader)) { return XtcError::READ_ERROR; @@ -545,7 +160,7 @@ XtcError XtcParser::readHeader() { XtcError XtcParser::readTitle() { constexpr auto titleOffset = 0x38; - if (!m_file.seek(titleOffset)) { + if (!m_file.seek64(titleOffset)) { return XtcError::READ_ERROR; } @@ -560,7 +175,7 @@ XtcError XtcParser::readTitle() { XtcError XtcParser::readAuthor() { // Read author as null-terminated UTF-8 string with max length 64, directly following title constexpr auto authorOffset = 0xB8; - if (!m_file.seek(authorOffset)) { + if (!m_file.seek64(authorOffset)) { return XtcError::READ_ERROR; } @@ -572,20 +187,83 @@ XtcError XtcParser::readAuthor() { return XtcError::OK; } -XtcError XtcParser::readChapters() { - m_hasChapters = false; - m_chapters.clear(); - m_chapters.shrink_to_fit(); +XtcError XtcParser::readFirstPageInfo() { + if (m_header.pageTableOffset == 0) { + LOG_DBG("XTC", "Page table offset is 0, cannot read"); + return XtcError::CORRUPTED_HEADER; + } - // Reopen the original file on demand because open() closes it after cache initialization. - if (!m_file.isOpen()) { - if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), m_file)) { - return XtcError::FILE_NOT_FOUND; - } + // Verify the file is large enough to contain the full page table + const uint64_t fileSize = m_file.size64(); + const uint64_t pageTableSize = static_cast(m_header.pageCount) * sizeof(PageTableEntry); + if (m_header.pageTableOffset < sizeof(XtcHeader) || m_header.pageTableOffset > fileSize || + pageTableSize > fileSize - m_header.pageTableOffset) { + LOG_DBG("XTC", "Page table exceeds file bounds"); + return XtcError::CORRUPTED_HEADER; + } + + // Read only the first entry to get default page dimensions + // All other entries are read on-demand via readPageTableEntry() + // This avoids allocating pageCount * 16 bytes (e.g. 65KB for 4000+ pages) + PageTableEntry entry; + if (!m_file.seek64(m_header.pageTableOffset)) { + LOG_DBG("XTC", "Failed to seek to page table at %llu", m_header.pageTableOffset); + return XtcError::READ_ERROR; + } + size_t bytesRead = m_file.read(reinterpret_cast(&entry), sizeof(PageTableEntry)); + if (bytesRead != sizeof(PageTableEntry)) { + LOG_DBG("XTC", "Failed to read first page table entry"); + return XtcError::READ_ERROR; + } + + m_defaultWidth = entry.width; + m_defaultHeight = entry.height; + + LOG_DBG("XTC", "Page table validated: %u pages, default %dx%d", m_header.pageCount, m_defaultWidth, m_defaultHeight); + return XtcError::OK; +} + +bool XtcParser::readPageTableEntry(uint32_t pageIndex, PageInfo& info) { + if (pageIndex >= m_header.pageCount) { + return false; + } + + if (!ensureFileOpen()) { + LOG_DBG("XTC", "Failed to reopen file for page table read"); + return false; + } + + // Seek to the specific page table entry on the SD card + const uint64_t entryOffset = m_header.pageTableOffset + static_cast(pageIndex) * sizeof(PageTableEntry); + if (!m_file.seek64(entryOffset)) { + LOG_DBG("XTC", "Failed to seek to page table entry %lu at %llu", pageIndex, entryOffset); + return false; + } + + PageTableEntry entry; + size_t bytesRead = m_file.read(reinterpret_cast(&entry), sizeof(PageTableEntry)); + if (bytesRead != sizeof(PageTableEntry)) { + LOG_DBG("XTC", "Failed to read page table entry %lu", pageIndex); + return false; + } + + info.offset = entry.dataOffset; + info.size = entry.dataSize; + info.width = entry.width; + info.height = entry.height; + info.bitDepth = m_bitDepth; + return true; +} + +XtcError XtcParser::readChapters() { + m_chapters.clear(); + + if (!ensureFileOpen()) { + return XtcError::READ_ERROR; } uint8_t hasChaptersFlag = 0; - if (!m_file.seek(0x0B)) { + if (!m_file.seek64(0x0B)) { return XtcError::READ_ERROR; } if (m_file.read(&hasChaptersFlag, sizeof(hasChaptersFlag)) != sizeof(hasChaptersFlag)) { @@ -597,7 +275,7 @@ XtcError XtcParser::readChapters() { } uint64_t chapterOffset = 0; - if (!m_file.seek(0x30)) { + if (!m_file.seek64(0x30)) { return XtcError::READ_ERROR; } if (m_file.read(reinterpret_cast(&chapterOffset), sizeof(chapterOffset)) != sizeof(chapterOffset)) { @@ -608,56 +286,36 @@ XtcError XtcParser::readChapters() { return XtcError::OK; } - const uint64_t fileSize = m_file.size(); - constexpr size_t chapterSize = 96; - - if (chapterOffset < sizeof(XtcHeader) || chapterOffset >= fileSize) { + const uint64_t fileSize = m_file.size64(); + if (chapterOffset < sizeof(XtcHeader) || chapterOffset >= fileSize || chapterOffset + 96 > fileSize) { return XtcError::OK; } - if (fileSize - chapterOffset < chapterSize) { - return XtcError::OK; - } - - uint64_t maxOffset = 0; - if (m_header.pageTableOffset > chapterOffset) { + // Clamp maxOffset to fileSize so bogus header values can't inflate chapterCount + uint64_t maxOffset = fileSize; + if (m_header.pageTableOffset > chapterOffset && m_header.pageTableOffset <= fileSize) { maxOffset = m_header.pageTableOffset; - } else if (m_header.dataOffset > chapterOffset) { + } else if (m_header.dataOffset > chapterOffset && m_header.dataOffset <= fileSize) { maxOffset = m_header.dataOffset; - } else { - maxOffset = fileSize; } if (maxOffset <= chapterOffset) { return XtcError::OK; } + constexpr size_t chapterSize = 96; const uint64_t available = maxOffset - chapterOffset; - const uint64_t chapterCount64 = available / chapterSize; - if (chapterCount64 == 0) { - return XtcError::OK; - } - - if (chapterCount64 > MAX_CHAPTERS || chapterCount64 > std::numeric_limits::max()) { - LOG_ERR("XTC", "Chapter table too large: available=%llu chapterCount=%llu", - static_cast(available), static_cast(chapterCount64)); - return XtcError::CORRUPTED_HEADER; - } - - const size_t chapterCount = static_cast(chapterCount64); + const size_t chapterCount = static_cast(available / chapterSize); if (chapterCount == 0) { return XtcError::OK; } - const size_t freeHeapBefore = ESP.getFreeHeap(); - const size_t maxAllocBefore = ESP.getMaxAllocHeap(); - - if (!seekToOffset(m_file, chapterOffset)) { + if (!m_file.seek64(chapterOffset)) { return XtcError::READ_ERROR; } - std::vector chapterBuf(chapterSize); m_chapters.reserve(chapterCount); + std::vector chapterBuf(chapterSize); for (size_t i = 0; i < chapterCount; i++) { if (m_file.read(chapterBuf.data(), chapterSize) != chapterSize) { return XtcError::READ_ERROR; @@ -701,29 +359,29 @@ XtcError XtcParser::readChapters() { m_chapters.push_back(std::move(chapter)); } - m_chapters.shrink_to_fit(); m_hasChapters = !m_chapters.empty(); - size_t chapterNameBytes = 0; - for (const auto& chapter : m_chapters) { - chapterNameBytes += chapter.name.capacity() + 1; - } - const size_t chapterVectorBytes = m_chapters.capacity() * sizeof(ChapterInfo); - const size_t totalChapterBytes = chapterVectorBytes + chapterNameBytes; - const size_t freeHeapAfter = ESP.getFreeHeap(); - const size_t maxAllocAfter = ESP.getMaxAllocHeap(); - const int heapDelta = static_cast(freeHeapBefore) - static_cast(freeHeapAfter); - const int maxAllocDelta = static_cast(maxAllocBefore) - static_cast(maxAllocAfter); - LOG_INF("XTC", "Chapter metadata: count=%u, vector~=%zu, names~=%zu, total~=%zu, heapDelta=%d, maxAllocDelta=%d", - static_cast(m_chapters.size()), chapterVectorBytes, chapterNameBytes, totalChapterBytes, - heapDelta, maxAllocDelta); + LOG_DBG("XTC", "Chapters: %u", static_cast(m_chapters.size())); return XtcError::OK; } const std::vector& XtcParser::getChapters() { - ensureChaptersLoaded(); + // Lazy load chapters on first access + if (!m_chaptersLoaded && m_hasChapters) { + const XtcError err = readChapters(); + if (err != XtcError::OK) { + LOG_ERR("XTC", "Failed to lazy-load chapters: %s", errorToString(err)); + m_hasChapters = false; + m_chapters.clear(); + } + m_chaptersLoaded = true; + // Close file after chapter read to free buffers for rendering + closeFile(); + } return m_chapters; } +bool XtcParser::getPageInfo(uint32_t pageIndex, PageInfo& info) { return readPageTableEntry(pageIndex, info); } + size_t XtcParser::loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSize) { if (!m_isOpen) { m_lastError = XtcError::FILE_NOT_FOUND; @@ -735,23 +393,20 @@ size_t XtcParser::loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSiz return 0; } - // Resolve the page location through the cache hierarchy before touching the data file. - PageInfo info; - if (!getPageInfo(pageIndex, info)) { + PageInfo page; + if (!readPageTableEntry(pageIndex, page)) { m_lastError = XtcError::READ_ERROR; return 0; } - // Reopen the source file lazily because normal parser open() does not keep it pinned. - if (!m_file.isOpen()) { - if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), m_file)) { - m_lastError = XtcError::FILE_NOT_FOUND; - return 0; - } + if (!ensureFileOpen()) { + m_lastError = XtcError::FILE_NOT_FOUND; + return 0; } - if (!seekToOffset(m_file, info.offset)) { - LOG_DBG("XTC", "Failed to seek to page %u at offset %llu", pageIndex, static_cast(info.offset)); + // Seek to page data + if (!m_file.seek64(page.offset)) { + LOG_DBG("XTC", "Failed to seek to page %u at offset %lu", pageIndex, page.offset); m_lastError = XtcError::READ_ERROR; return 0; } @@ -785,14 +440,14 @@ size_t XtcParser::loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSiz bitmapSize = ((pageHeader.width + 7) / 8) * pageHeader.height; } - // The caller owns the buffer, so fail early if it is too small. + // Check buffer size if (bufferSize < bitmapSize) { LOG_DBG("XTC", "Buffer too small: need %u, have %u", bitmapSize, bufferSize); m_lastError = XtcError::MEMORY_ERROR; return 0; } - // Read the bitmap payload into the caller-provided buffer. + // Read bitmap data size_t bytesRead = m_file.read(buffer, bitmapSize); if (bytesRead != bitmapSize) { LOG_DBG("XTC", "Page read error: expected %u, got %u", bitmapSize, bytesRead); @@ -815,25 +470,21 @@ XtcError XtcParser::loadPageStreaming(uint32_t pageIndex, return XtcError::PAGE_OUT_OF_RANGE; } - // Streaming uses the same cache lookup path but reads the payload in chunks. - PageInfo info; - if (!getPageInfo(pageIndex, info)) { + PageInfo page; + if (!readPageTableEntry(pageIndex, page)) { return XtcError::READ_ERROR; } - // Reopen the source file on demand for streaming reads as well. - if (!m_file.isOpen()) { - if (!Storage.openFileForRead("XTC", m_originalPath.c_str(), m_file)) { - return XtcError::FILE_NOT_FOUND; - } + if (!ensureFileOpen()) { + return XtcError::FILE_NOT_FOUND; } - if (!seekToOffset(m_file, info.offset)) { - LOG_DBG("XTC", "Failed to seek to page %u at offset %llu", pageIndex, static_cast(info.offset)); + // Seek to page data + if (!m_file.seek64(page.offset)) { return XtcError::READ_ERROR; } - // Read and validate the page header before yielding any bitmap bytes. + // Read and skip page header (XTG for 1-bit, XTH for 2-bit) XtgPageHeader pageHeader; size_t headerRead = m_file.read(reinterpret_cast(&pageHeader), sizeof(XtgPageHeader)); const uint32_t expectedMagic = (m_bitDepth == 2) ? XTH_MAGIC : XTG_MAGIC; @@ -844,7 +495,6 @@ XtcError XtcParser::loadPageStreaming(uint32_t pageIndex, // Calculate bitmap size based on bit depth // XTG (1-bit): Row-major, ((width+7)/8) * height bytes // XTH (2-bit): Two bit planes, ((width * height + 7) / 8) * 2 bytes - // Match the bitmap sizing rules used by the non-streaming path. size_t bitmapSize; if (m_bitDepth == 2) { bitmapSize = ((static_cast(pageHeader.width) * pageHeader.height + 7) / 8) * 2; @@ -852,7 +502,7 @@ XtcError XtcParser::loadPageStreaming(uint32_t pageIndex, bitmapSize = ((pageHeader.width + 7) / 8) * pageHeader.height; } - // Feed the bitmap to the callback in bounded chunks to keep peak memory low. + // Read in chunks std::vector chunk(chunkSize); size_t totalRead = 0; @@ -888,4 +538,4 @@ bool XtcParser::isValidXtcFile(const char* filepath) { return (magic == XTC_MAGIC || magic == XTCH_MAGIC); } -} // namespace xtc +} // namespace xtc \ No newline at end of file diff --git a/lib/Xtc/Xtc/XtcParser.h b/lib/Xtc/Xtc/XtcParser.h index 38e00862..b688d793 100644 --- a/lib/Xtc/Xtc/XtcParser.h +++ b/lib/Xtc/Xtc/XtcParser.h @@ -9,8 +9,6 @@ #include -#include -#include #include #include #include @@ -25,6 +23,9 @@ namespace xtc { * * Reads XTC files from SD card and extracts page data. * Designed for ESP32-C3's limited RAM (~380KB) using streaming. + * + * The source file is kept closed between reads to free heap for rendering. + * It is reopened on-demand for page table lookups and bitmap data reads. */ class XtcParser { public: @@ -32,7 +33,7 @@ class XtcParser { ~XtcParser(); // File open/close - XtcError open(const char* filepath, const char* cacheDir); + XtcError open(const char* filepath); void close(); bool isOpen() const { return m_isOpen; } @@ -43,14 +44,28 @@ class XtcParser { uint16_t getHeight() const { return m_defaultHeight; } uint8_t getBitDepth() const { return m_bitDepth; } // 1 = XTC/XTG, 2 = XTCH/XTH - // Page information - three-tier cache interface + // Page information bool getPageInfo(uint32_t pageIndex, PageInfo& info); - // Preload window around specified page (optimize sequential page turns) - void prefetchWindow(uint32_t pageIndex); - - // Load page bitmap (unchanged) + /** + * Load page bitmap (raw 1-bit data, skipping XTG header) + * + * @param pageIndex Page index (0-based) + * @param buffer Output buffer (caller allocated) + * @param bufferSize Buffer size + * @return Number of bytes read on success, 0 on failure + */ size_t loadPage(uint32_t pageIndex, uint8_t* buffer, size_t bufferSize); + + /** + * Streaming page load + * Memory-efficient method that reads page data in chunks. + * + * @param pageIndex Page index + * @param callback Callback function to receive data chunks + * @param chunkSize Chunk size (default: 1024 bytes) + * @return Error code + */ XtcError loadPageStreaming(uint32_t pageIndex, std::function callback, size_t chunkSize = 1024); @@ -70,62 +85,30 @@ class XtcParser { private: FsFile m_file; - FsFile m_cacheFile; + std::string m_filepath; bool m_isOpen; XtcHeader m_header; - std::string m_cacheDir; - std::string m_cacheFilePath; - std::string m_originalPath; + std::vector m_chapters; std::string m_title; std::string m_author; uint16_t m_defaultWidth; uint16_t m_defaultHeight; - uint8_t m_bitDepth; + uint8_t m_bitDepth; // 1 = XTC/XTG (1-bit), 2 = XTCH/XTH (2-bit) bool m_hasChapters; - bool m_chaptersLoaded = false; + bool m_chaptersLoaded; XtcError m_lastError; - uint32_t m_accessCounter = 0; - - // L1: Hot cache (fixed 4 entries) - std::array m_l1Cache; - - // L2: Sliding window (fixed size array) - std::array m_l2Window; - uint32_t m_l2WindowStart = 0; - size_t m_l2WindowCount = 0; - bool m_l2Valid = false; - - // Chapters (usually few, keep in memory) - std::vector m_chapters; - - // Original Page Table offset (for rebuilding cache) - uint64_t m_pageTableOffset = 0; // Internal helper functions XtcError readHeader(); + XtcError readFirstPageInfo(); XtcError readTitle(); XtcError readAuthor(); XtcError readChapters(); - void ensureChaptersLoaded(); + bool readPageTableEntry(uint32_t pageIndex, PageInfo& info); - // L3 cache management - bool isPageTableCacheValid() const; - XtcError buildPageTableCache(); - bool openCacheFile(); - void closeCacheFile(); - - // L1/L2 cache operations - bool lookupL1(uint32_t pageIndex, PageInfo& info); - void updateL1(uint32_t pageIndex, const PageInfo& info); - bool lookupL2(uint32_t pageIndex, PageInfo& info); - void loadL2Window(uint32_t centerPage); - - // Safe deserialization (alignment-safe for ESP32-C3) - static void safeDeserializeHeader(const uint8_t* buf, PageTableCacheHeader& header); - static void safeSerializeHeader(uint8_t* buf, const PageTableCacheHeader& header); - - // Utility functions - uint32_t calculateFileHash(const char* filepath) const; + // File handle management — reopen on demand, close after use + bool ensureFileOpen(); + void closeFile(); }; } // namespace xtc diff --git a/lib/Xtc/Xtc/XtcTypes.h b/lib/Xtc/Xtc/XtcTypes.h index b86c8934..83dfd340 100644 --- a/lib/Xtc/Xtc/XtcTypes.h +++ b/lib/Xtc/Xtc/XtcTypes.h @@ -102,33 +102,6 @@ struct ChapterInfo { uint16_t endPage; }; -// Cache configuration -constexpr size_t L1_CACHE_SIZE = 4; // L1 cache entries -constexpr size_t L2_WINDOW_SIZE = 100; // L2 window size (reduced for 2-bit memory) -constexpr uint32_t PAGE_TABLE_CACHE_VERSION = 1; // Cache file version - -// Cache magic number -constexpr uint32_t PAGE_TABLE_CACHE_MAGIC = 0x50435458; // "XTCP" - -// Cache file header - NOTE: Do NOT read directly from file buffer! -// Use safeDeserializeHeader() function for alignment-safe access. -struct PageTableCacheHeader { - uint32_t magic; // 'XTCP' = 0x50435458 - uint32_t version; // Cache version - uint32_t pageCount; // Total pages - uint32_t originalHash; // Original file hash (for validation) - uint64_t originalSize; // Original file size (for validation) - uint32_t entrySize; // PageInfo size (16) - uint32_t reserved; // Reserved -}; - -// L1 cache entry -struct L1CacheEntry { - uint32_t pageIndex = 0xFFFFFFFF; // 0xFFFFFFFF = invalid - PageInfo info{}; - uint32_t lastAccess = 0; // Timestamp for LRU -}; - // Error codes enum class XtcError { OK = 0, diff --git a/lib/hal/HalStorage.cpp b/lib/hal/HalStorage.cpp index 43c4b917..30165fbd 100644 --- a/lib/hal/HalStorage.cpp +++ b/lib/hal/HalStorage.cpp @@ -183,13 +183,13 @@ bool HalStorage::copyFile(const char* moduleName, const std::string& srcPath, co void HalFile::flush() { HAL_FILE_WRAPPED_CALL(flush, ); } size_t HalFile::getName(char* name, size_t len) { HAL_FILE_WRAPPED_CALL(getName, name, len); } -size_t HalFile::size() { HAL_FILE_FORWARD_CALL(size, ); } // already thread-safe, no need to wrap -size_t HalFile::fileSize() { HAL_FILE_FORWARD_CALL(fileSize, ); } // already thread-safe, no need to wrap +size_t HalFile::size() { assert(impl != nullptr); return static_cast(impl->file.size()); } +size_t HalFile::fileSize() { assert(impl != nullptr); return static_cast(impl->file.fileSize()); } bool HalFile::seek(size_t pos) { HAL_FILE_WRAPPED_CALL(seekSet, pos); } bool HalFile::seekCur(int64_t offset) { HAL_FILE_WRAPPED_CALL(seekCur, offset); } bool HalFile::seekSet(size_t offset) { HAL_FILE_WRAPPED_CALL(seekSet, offset); } int HalFile::available() const { HAL_FILE_WRAPPED_CALL(available, ); } -size_t HalFile::position() const { HAL_FILE_WRAPPED_CALL(position, ); } +size_t HalFile::position() const { assert(impl != nullptr); return static_cast(impl->file.position()); } int HalFile::read(void* buf, size_t count) { HAL_FILE_WRAPPED_CALL(read, buf, count); } int HalFile::read() { HAL_FILE_WRAPPED_CALL(read, ); } size_t HalFile::write(const void* buf, size_t count) { HAL_FILE_WRAPPED_CALL(write, buf, count); } @@ -201,6 +201,11 @@ bool HalFile::getModifyDateTime(uint16_t* pdate, uint16_t* ptime) { bool HalFile::isDirectory() const { HAL_FILE_FORWARD_CALL(isDirectory, ); } // already thread-safe, no need to wrap void HalFile::rewindDirectory() { HAL_FILE_WRAPPED_CALL(rewindDirectory, ); } bool HalFile::close() { HAL_FILE_WRAPPED_CALL(close, ); } +uint64_t HalFile::size64() { HAL_FILE_FORWARD_CALL(size, ); } +uint64_t HalFile::fileSize64() { HAL_FILE_FORWARD_CALL(fileSize, ); } +bool HalFile::seek64(uint64_t pos) { HAL_FILE_WRAPPED_CALL(seekSet, pos); } +bool HalFile::seekSet64(uint64_t offset) { HAL_FILE_WRAPPED_CALL(seekSet, offset); } +uint64_t HalFile::position64() const { HAL_FILE_FORWARD_CALL(position, ); } HalFile HalFile::openNextFile() { HalStorage::StorageLock lock; assert(impl != nullptr); diff --git a/lib/hal/HalStorage.h b/lib/hal/HalStorage.h index 4a59b37e..da76a55b 100644 --- a/lib/hal/HalStorage.h +++ b/lib/hal/HalStorage.h @@ -4,6 +4,7 @@ #include // for oflag_t #include +#include #include #include #include @@ -88,6 +89,12 @@ class HalFile : public Print { size_t position() const; int read(void* buf, size_t count); int read(); // read a single byte + + uint64_t size64(); + uint64_t fileSize64(); + bool seek64(uint64_t pos); + bool seekSet64(uint64_t offset); + uint64_t position64() const; size_t write(const void* buf, size_t count); size_t write(uint8_t b) override; bool rename(const char* newPath); diff --git a/src/activities/reader/XtcReaderActivity.cpp b/src/activities/reader/XtcReaderActivity.cpp index 7b69a78f..d5a631fc 100644 --- a/src/activities/reader/XtcReaderActivity.cpp +++ b/src/activities/reader/XtcReaderActivity.cpp @@ -47,8 +47,6 @@ void XtcReaderActivity::onEnter() { APP_STATE.saveToFile(); RECENT_BOOKS.addBook(xtc->getPath(), xtc->getTitle(), xtc->getAuthor(), "", xtc->getThumbBmpPath()); - xtc->prefetchPages(currentPage); - // Trigger first update requestUpdate(); } @@ -75,7 +73,6 @@ void XtcReaderActivity::loop() { [this](const ActivityResult& result) { if (!result.isCancelled) { currentPage = std::get(result.data).page; - xtc->prefetchPages(currentPage); } }); } @@ -133,14 +130,12 @@ void XtcReaderActivity::loop() { } else { currentPage = 0; } - xtc->prefetchPages(currentPage); requestUpdate(); } else if (nextTriggered) { currentPage += skipAmount; if (currentPage >= xtc->getPageCount()) { currentPage = xtc->getPageCount(); // Allow showing "End of book" } - xtc->prefetchPages(currentPage); requestUpdate(); } }