fix: increase loadable epub size (#1638)
## Summary * **What is the goal of this PR?** Slightly increase the OOM limit when loading epubs * **What changes are included?** Switched vectors for parsing epubs to deques, allowing use of more memory ## Additional Context * Increases loadable epub size from 2000+ chapter/ToC entries loadable to 5000+ chapter/ToC entries * #1574, but without the complicated stuff --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**NO**_ ### Testing | Commit | Book | Time | |----------|------|-------| |83cd96bc2f| 1000.epub | ~30 sec | |83cd96bc2f| 5000.epub | crash | | PR | 1000.epub | ~29 sec | | PR | 5000.epub | ~2 min 20 sec | => No actual loading time regressions [tested_epubs.zip](https://github.com/user-attachments/files/26645243/tested_epubs.zip)
This commit is contained in:
@@ -4,7 +4,7 @@
|
|||||||
#include <Serialization.h>
|
#include <Serialization.h>
|
||||||
#include <ZipFile.h>
|
#include <ZipFile.h>
|
||||||
|
|
||||||
#include <vector>
|
#include <deque>
|
||||||
|
|
||||||
#include "FsHelpers.h"
|
#include "FsHelpers.h"
|
||||||
|
|
||||||
@@ -50,7 +50,7 @@ bool BookMetadataCache::beginTocPass() {
|
|||||||
|
|
||||||
if (spineCount >= LARGE_SPINE_THRESHOLD) {
|
if (spineCount >= LARGE_SPINE_THRESHOLD) {
|
||||||
spineHrefIndex.clear();
|
spineHrefIndex.clear();
|
||||||
spineHrefIndex.reserve(spineCount);
|
spineHrefIndex.resize(spineCount);
|
||||||
spineFile.seek(0);
|
spineFile.seek(0);
|
||||||
for (int i = 0; i < spineCount; i++) {
|
for (int i = 0; i < spineCount; i++) {
|
||||||
auto entry = readSpineEntry(spineFile);
|
auto entry = readSpineEntry(spineFile);
|
||||||
@@ -58,7 +58,7 @@ bool BookMetadataCache::beginTocPass() {
|
|||||||
idx.hrefHash = fnvHash64(entry.href);
|
idx.hrefHash = fnvHash64(entry.href);
|
||||||
idx.hrefLen = static_cast<uint16_t>(entry.href.size());
|
idx.hrefLen = static_cast<uint16_t>(entry.href.size());
|
||||||
idx.spineIndex = static_cast<int16_t>(i);
|
idx.spineIndex = static_cast<int16_t>(i);
|
||||||
spineHrefIndex.push_back(idx);
|
spineHrefIndex[i] = idx;
|
||||||
}
|
}
|
||||||
std::sort(spineHrefIndex.begin(), spineHrefIndex.end(),
|
std::sort(spineHrefIndex.begin(), spineHrefIndex.end(),
|
||||||
[](const SpineHrefIndexEntry& a, const SpineHrefIndexEntry& b) {
|
[](const SpineHrefIndexEntry& a, const SpineHrefIndexEntry& b) {
|
||||||
@@ -153,7 +153,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
|
|||||||
// Loop through spines from spine file matching up TOC indexes, calculating cumulative size and writing to book.bin
|
// Loop through spines from spine file matching up TOC indexes, calculating cumulative size and writing to book.bin
|
||||||
|
|
||||||
// Build spineIndex->tocIndex mapping in one pass (O(n) instead of O(n*m))
|
// Build spineIndex->tocIndex mapping in one pass (O(n) instead of O(n*m))
|
||||||
std::vector<int16_t> spineToTocIndex(spineCount, -1);
|
std::deque<int16_t> spineToTocIndex(spineCount, -1);
|
||||||
tocFile.seek(0);
|
tocFile.seek(0);
|
||||||
for (int j = 0; j < tocCount; j++) {
|
for (int j = 0; j < tocCount; j++) {
|
||||||
auto tocEntry = readTocEntry(tocFile);
|
auto tocEntry = readTocEntry(tocFile);
|
||||||
@@ -181,14 +181,14 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
|
|||||||
// This is O(n*log(m)) instead of O(n*m) while avoiding memory exhaustion.
|
// This is O(n*log(m)) instead of O(n*m) while avoiding memory exhaustion.
|
||||||
// See: https://github.com/crosspoint-reader/crosspoint-reader/issues/134
|
// See: https://github.com/crosspoint-reader/crosspoint-reader/issues/134
|
||||||
|
|
||||||
std::vector<uint32_t> spineSizes;
|
std::deque<uint32_t> spineSizes;
|
||||||
bool useBatchSizes = false;
|
bool useBatchSizes = false;
|
||||||
|
|
||||||
if (spineCount >= LARGE_SPINE_THRESHOLD) {
|
if (spineCount >= LARGE_SPINE_THRESHOLD) {
|
||||||
LOG_DBG("BMC", "Using batch size lookup for %d spine items", spineCount);
|
LOG_DBG("BMC", "Using batch size lookup for %d spine items", spineCount);
|
||||||
|
|
||||||
std::vector<ZipFile::SizeTarget> targets;
|
std::deque<ZipFile::SizeTarget> targets;
|
||||||
targets.reserve(spineCount);
|
targets.resize(spineCount);
|
||||||
|
|
||||||
spineFile.seek(0);
|
spineFile.seek(0);
|
||||||
for (int i = 0; i < spineCount; i++) {
|
for (int i = 0; i < spineCount; i++) {
|
||||||
@@ -199,7 +199,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
|
|||||||
t.hash = ZipFile::fnvHash64(path.c_str(), path.size());
|
t.hash = ZipFile::fnvHash64(path.c_str(), path.size());
|
||||||
t.len = static_cast<uint16_t>(path.size());
|
t.len = static_cast<uint16_t>(path.size());
|
||||||
t.index = static_cast<uint16_t>(i);
|
t.index = static_cast<uint16_t>(i);
|
||||||
targets.push_back(t);
|
targets[i] = t;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::sort(targets.begin(), targets.end(), [](const ZipFile::SizeTarget& a, const ZipFile::SizeTarget& b) {
|
std::sort(targets.begin(), targets.end(), [](const ZipFile::SizeTarget& a, const ZipFile::SizeTarget& b) {
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
#include <HalStorage.h>
|
#include <HalStorage.h>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <deque>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
class BookMetadataCache {
|
class BookMetadataCache {
|
||||||
public:
|
public:
|
||||||
@@ -61,7 +61,7 @@ class BookMetadataCache {
|
|||||||
uint16_t hrefLen; // length for collision reduction
|
uint16_t hrefLen; // length for collision reduction
|
||||||
int16_t spineIndex;
|
int16_t spineIndex;
|
||||||
};
|
};
|
||||||
std::vector<SpineHrefIndexEntry> spineHrefIndex;
|
std::deque<SpineHrefIndexEntry> spineHrefIndex;
|
||||||
bool useSpineHrefIndex = false;
|
bool useSpineHrefIndex = false;
|
||||||
|
|
||||||
static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400;
|
static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#include <Print.h>
|
#include <Print.h>
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <deque>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Epub.h"
|
#include "Epub.h"
|
||||||
@@ -37,7 +38,7 @@ class ContentOpfParser final : public Print {
|
|||||||
uint16_t idLen; // length for collision reduction
|
uint16_t idLen; // length for collision reduction
|
||||||
uint32_t fileOffset; // offset in .items.bin
|
uint32_t fileOffset; // offset in .items.bin
|
||||||
};
|
};
|
||||||
std::vector<ItemIndexEntry> itemIndex;
|
std::deque<ItemIndexEntry> itemIndex;
|
||||||
bool useItemIndex = false;
|
bool useItemIndex = false;
|
||||||
|
|
||||||
static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400;
|
static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400;
|
||||||
|
|||||||
@@ -295,7 +295,7 @@ bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int ZipFile::fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes) {
|
int ZipFile::fillUncompressedSizes(std::deque<SizeTarget>& targets, std::deque<uint32_t>& sizes) {
|
||||||
if (targets.empty()) {
|
if (targets.empty()) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
#include <HalStorage.h>
|
#include <HalStorage.h>
|
||||||
|
|
||||||
|
#include <deque>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
class ZipFile {
|
class ZipFile {
|
||||||
public:
|
public:
|
||||||
@@ -64,7 +64,7 @@ class ZipFile {
|
|||||||
// Batch lookup: scan ZIP central dir once and fill sizes for matching targets.
|
// Batch lookup: scan ZIP central dir once and fill sizes for matching targets.
|
||||||
// targets must be sorted by (hash, len). sizes[target.index] receives uncompressedSize.
|
// targets must be sorted by (hash, len). sizes[target.index] receives uncompressedSize.
|
||||||
// Returns number of targets matched.
|
// Returns number of targets matched.
|
||||||
int fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes);
|
int fillUncompressedSizes(std::deque<SizeTarget>& targets, std::deque<uint32_t>& sizes);
|
||||||
// Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple
|
// Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple
|
||||||
// These functions will open and close the zip as needed
|
// These functions will open and close the zip as needed
|
||||||
uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);
|
uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);
|
||||||
|
|||||||
Reference in New Issue
Block a user