Deduplicate identical CSS files in EPUB parsing & probe images for dimensions instead of reading the full file
Some EPUB converters emit byte-identical stylesheets per chapter (100+ entries). Now scan the ZIP central directory once to identify duplicates by CRC32 and compressed size, then skip parsing identical files. This avoids redundant ZIP lookups and SD extraction round-trips while preserving all styles since rules merge into a global set. Also add extractItemToFile helper and allowEarlyStop parameter to readItemContentsToStream.
This commit is contained in:
+18
-3
@@ -69,7 +69,10 @@ class ZipFile {
|
||||
// Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple
|
||||
// These functions will open and close the zip as needed
|
||||
uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);
|
||||
bool readFileToStream(const char* filename, Print& out, size_t chunkSize);
|
||||
// allowEarlyStop: a short write from `out` is treated as the sink asking to
|
||||
// stop (returns true) instead of a write failure — used by header probes
|
||||
// that only need the first bytes of an entry.
|
||||
bool readFileToStream(const char* filename, Print& out, size_t chunkSize, bool allowEarlyStop = false);
|
||||
|
||||
template <typename F>
|
||||
bool enumerateFilePaths(F&& callback) {
|
||||
@@ -80,6 +83,14 @@ class ZipFile {
|
||||
return true;
|
||||
}
|
||||
|
||||
return enumerateFileEntries([&callback](std::string_view path, uint32_t, uint32_t) { callback(path); });
|
||||
}
|
||||
|
||||
// Callback receives (path, crc32, compressedSize) for each central-directory
|
||||
// entry. Always scans the central directory: the slim-stat cache does not
|
||||
// hold CRCs.
|
||||
template <typename F>
|
||||
bool enumerateFileEntries(F&& callback) {
|
||||
const bool wasOpen = isOpen();
|
||||
if (!wasOpen && !open()) {
|
||||
return false;
|
||||
@@ -103,7 +114,11 @@ class ZipFile {
|
||||
break;
|
||||
}
|
||||
|
||||
file.seekCur(24);
|
||||
file.seekCur(12);
|
||||
uint32_t crc32, compressedSize;
|
||||
file.read(&crc32, 4);
|
||||
file.read(&compressedSize, 4);
|
||||
file.seekCur(4);
|
||||
uint16_t nameLen, m, k;
|
||||
file.read(&nameLen, 2);
|
||||
file.read(&m, 2);
|
||||
@@ -113,7 +128,7 @@ class ZipFile {
|
||||
if (nameLen < sizeof(itemName)) {
|
||||
file.read(itemName, nameLen);
|
||||
itemName[nameLen] = '\0';
|
||||
callback(std::string_view{itemName, nameLen});
|
||||
callback(std::string_view{itemName, nameLen}, crc32, compressedSize);
|
||||
} else {
|
||||
file.seekCur(nameLen);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user