fix: avoid zip-wide css scan for large epubs (#2213)
## Summary * **What is the goal of this PR?** * This PR ports over Crossink's handling of large EPUBs that helps prevent crashes during open after the book metadata cache is built. * **What changes are included?** * Removes the post-indexing ZIP-wide CSS discovery pass that built an in-memory map of every ZIP entry. * Reuses `content.opf` parsing to collect declared CSS files without writing spine entries again. * Temporarily releases the loaded book metadata cache while rebuilding CSS for cached books. * Parses CSS before reloading `book.bin` after a fresh cache build, leaving more heap available during CSS rule parsing. ## Additional Context * User reported their EPUB opening fine on Crossink but would crash on Crosspoint. Verified this claim on my own devices. * The crash this addresses happened after `book.bin` was successfully built, when CSS discovery allocated a large `unordered_map` for ~3k EPUB ZIP entries. * Tradeoff: CSS files not declared in `content.opf` are no longer discovered by scanning the full ZIP. This avoids the high-risk memory allocation but improperly formatted EPUBs (ones that don't declare their CSS styles in `content.opf` will render without styling and fallback to inline styles. * User provided epub that was crashing prior to this change: https://www.mediafire.com/file/g57ea4mj13iunvh/Quang+%C3%82m+Chi+Ngo%E1%BA%A1i+-+Nh%C4%A9+C%C4%83n.epub/file --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**< YES >**_
This commit is contained in:
Julia
+33
-36
@@ -44,7 +44,7 @@ bool Epub::findContentOpfFile(std::string* contentOpfFile) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
|
||||
bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, const bool writeSpineEntries) {
|
||||
std::string contentOpfFilePath;
|
||||
if (!findContentOpfFile(&contentOpfFilePath)) {
|
||||
LOG_ERR("EBP", "Could not find content.opf in zip");
|
||||
@@ -61,7 +61,8 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize, bookMetadataCache.get());
|
||||
ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize,
|
||||
writeSpineEntries ? bookMetadataCache.get() : nullptr);
|
||||
if (!opfParser.setup()) {
|
||||
LOG_ERR("EBP", "Could not setup content.opf parser");
|
||||
return false;
|
||||
@@ -256,35 +257,27 @@ bool Epub::parseTocNavFile() const {
|
||||
}
|
||||
|
||||
void Epub::discoverCssFilesFromZip() {
|
||||
if (!bookMetadataCache || !bookMetadataCache->isLoaded()) {
|
||||
LOG_ERR("EBP", "Cannot discover CSS from ZIP because book metadata cache is not loaded");
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string& opfDir = contentBasePath;
|
||||
ZipFile zf(filepath);
|
||||
|
||||
if (!zf.loadAllFileStatSlims()) {
|
||||
LOG_ERR("EBP", "Failed to load ZIP file stat slims for CSS discovery");
|
||||
return;
|
||||
}
|
||||
if (!zf.enumerateFilePaths([&](std::string_view filePath) {
|
||||
if (!opfDir.empty() && filePath.find(opfDir) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t lastSlash = contentBasePath.find_last_of('/');
|
||||
if (!FsHelpers::hasCssExtension(filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::string opfDir = (lastSlash != std::string::npos) ? contentBasePath.substr(0, lastSlash + 1) : "";
|
||||
if (std::find(cssFiles.begin(), cssFiles.end(), filePath) != cssFiles.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
zf.enumerateFilePaths([&](std::string_view filePath) {
|
||||
if (!opfDir.empty() && filePath.find(opfDir) != 0) {
|
||||
return; // Skip files that are not in the same directory as OPF manifest, as CSS files are typically located
|
||||
// there or in subfolders
|
||||
}
|
||||
|
||||
if (FsHelpers::hasCssExtension(filePath)) {
|
||||
if (std::find(cssFiles.begin(), cssFiles.end(), filePath) == cssFiles.end()) {
|
||||
LOG_DBG("EBP", "Discovered CSS file via ZIP enumeration: %.*s", (int)filePath.size(), filePath.data());
|
||||
cssFiles.push_back(std::string{filePath});
|
||||
}
|
||||
}
|
||||
});
|
||||
})) {
|
||||
LOG_ERR("EBP", "Failed to enumerate ZIP file paths for CSS discovery");
|
||||
}
|
||||
}
|
||||
|
||||
void Epub::parseCssFiles() const {
|
||||
@@ -383,15 +376,20 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
|
||||
LOG_DBG("EBP", "CSS rules cache missing or stale, attempting to parse CSS files");
|
||||
cssParser->deleteCache();
|
||||
|
||||
if (!parseContentOpf(bookMetadataCache->coreMetadata)) {
|
||||
BookMetadataCache::BookMetadata cachedMetadata = bookMetadataCache->coreMetadata;
|
||||
if (!parseContentOpf(cachedMetadata, /*writeSpineEntries=*/false)) {
|
||||
LOG_ERR("EBP", "Could not parse content.opf from cached bookMetadata for CSS files");
|
||||
// continue anyway - book will work without CSS and we'll still load any inline style CSS
|
||||
} else {
|
||||
// Handle case where CSS files are not listed in OPF manifest
|
||||
// but are still referenced by HTML files - discover and parse them too
|
||||
discoverCssFilesFromZip();
|
||||
}
|
||||
bookMetadataCache.reset();
|
||||
parseCssFiles();
|
||||
bookMetadataCache.reset(new BookMetadataCache(cachePath));
|
||||
if (!bookMetadataCache->load()) {
|
||||
LOG_ERR("EBP", "Failed to reload cache after CSS rebuild");
|
||||
return false;
|
||||
}
|
||||
// Invalidate section caches so they are rebuilt with the new CSS
|
||||
Storage.removeDir((cachePath + "/sections").c_str());
|
||||
}
|
||||
@@ -428,6 +426,7 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
|
||||
LOG_ERR("EBP", "Could not parse content.opf");
|
||||
return false;
|
||||
}
|
||||
discoverCssFilesFromZip();
|
||||
if (!bookMetadataCache->endContentOpfPass()) {
|
||||
LOG_ERR("EBP", "Could not end writing content.opf pass");
|
||||
return false;
|
||||
@@ -485,6 +484,13 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
|
||||
LOG_DBG("EBP", "Could not cleanup tmp files - ignoring");
|
||||
}
|
||||
|
||||
if (!skipLoadingCss) {
|
||||
// Parse CSS before reloading book.bin to leave more heap for CSS rule-table growth.
|
||||
bookMetadataCache.reset();
|
||||
parseCssFiles();
|
||||
Storage.removeDir((cachePath + "/sections").c_str());
|
||||
}
|
||||
|
||||
// Reload the cache from disk so it's in the correct state
|
||||
bookMetadataCache.reset(new BookMetadataCache(cachePath));
|
||||
if (!bookMetadataCache->load()) {
|
||||
@@ -492,15 +498,6 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!skipLoadingCss) {
|
||||
// Handle case where CSS files are not listed in OPF manifest
|
||||
// but are still referenced by HTML files - discover and parse them too
|
||||
discoverCssFilesFromZip();
|
||||
// Parse CSS files after cache reload
|
||||
parseCssFiles();
|
||||
Storage.removeDir((cachePath + "/sections").c_str());
|
||||
}
|
||||
|
||||
LOG_DBG("EBP", "Loaded ePub: %s", filepath.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
+2
-2
@@ -31,11 +31,11 @@ class Epub {
|
||||
std::vector<std::string> cssFiles;
|
||||
|
||||
bool findContentOpfFile(std::string* contentOpfFile) const;
|
||||
bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata);
|
||||
bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, bool writeSpineEntries = true);
|
||||
bool parseTocNcxFile() const;
|
||||
bool parseTocNavFile() const;
|
||||
void parseCssFiles() const;
|
||||
void discoverCssFilesFromZip();
|
||||
void parseCssFiles() const;
|
||||
|
||||
public:
|
||||
explicit Epub(std::string filepath, const std::string& cacheDir) : filepath(std::move(filepath)) {
|
||||
|
||||
+52
-3
@@ -72,9 +72,58 @@ class ZipFile {
|
||||
bool readFileToStream(const char* filename, Print& out, size_t chunkSize);
|
||||
|
||||
template <typename F>
|
||||
void enumerateFilePaths(F&& callback) const {
|
||||
for (const auto& entry : fileStatSlimCache) {
|
||||
callback(std::string_view{entry.first});
|
||||
bool enumerateFilePaths(F&& callback) {
|
||||
if (!fileStatSlimCache.empty()) {
|
||||
for (const auto& entry : fileStatSlimCache) {
|
||||
callback(std::string_view{entry.first});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const bool wasOpen = isOpen();
|
||||
if (!wasOpen && !open()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!loadZipDetails()) {
|
||||
if (!wasOpen) {
|
||||
close();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
file.seek(zipDetails.centralDirOffset);
|
||||
|
||||
uint32_t sig;
|
||||
char itemName[256];
|
||||
|
||||
while (file.available()) {
|
||||
file.read(&sig, 4);
|
||||
if (sig != 0x02014b50) {
|
||||
break;
|
||||
}
|
||||
|
||||
file.seekCur(24);
|
||||
uint16_t nameLen, m, k;
|
||||
file.read(&nameLen, 2);
|
||||
file.read(&m, 2);
|
||||
file.read(&k, 2);
|
||||
file.seekCur(12);
|
||||
|
||||
if (nameLen < sizeof(itemName)) {
|
||||
file.read(itemName, nameLen);
|
||||
itemName[nameLen] = '\0';
|
||||
callback(std::string_view{itemName, nameLen});
|
||||
} else {
|
||||
file.seekCur(nameLen);
|
||||
}
|
||||
|
||||
file.seekCur(m + k);
|
||||
}
|
||||
|
||||
if (!wasOpen) {
|
||||
close();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user