Merge pull request #63 from jpirnay/refactor-deque

refactor: Adapt upstream PR 1638 - use deque to increase usable epub size

lib/Epub/Epub/BookMetadataCache.cpp

@@ -4,7 +4,7 @@
 #include <Serialization.h>
 #include <ZipFile.h>
 
-#include <vector>
+#include <deque>
 
 #include "FsHelpers.h"
 
@@ -50,7 +50,7 @@ bool BookMetadataCache::beginTocPass() {
 
   if (spineCount >= LARGE_SPINE_THRESHOLD) {
     spineHrefIndex.clear();
-    spineHrefIndex.reserve(spineCount);
+    spineHrefIndex.resize(spineCount);
     spineFile.seek(0);
     for (int i = 0; i < spineCount; i++) {
       auto entry = readSpineEntry(spineFile);
@@ -58,7 +58,7 @@ bool BookMetadataCache::beginTocPass() {
       idx.hrefHash = fnvHash64(entry.href);
       idx.hrefLen = static_cast<uint16_t>(entry.href.size());
       idx.spineIndex = static_cast<int16_t>(i);
-      spineHrefIndex.push_back(idx);
+      spineHrefIndex[i] = idx;
     }
     std::sort(spineHrefIndex.begin(), spineHrefIndex.end(),
               [](const SpineHrefIndexEntry& a, const SpineHrefIndexEntry& b) {
@@ -157,7 +157,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
   // Loop through spines from spine file matching up TOC indexes, calculating cumulative size and writing to book.bin
 
   // Build spineIndex->tocIndex mapping in one pass (O(n) instead of O(n*m))
-  std::vector<int16_t> spineToTocIndex(spineCount, -1);
+  std::deque<int16_t> spineToTocIndex(spineCount, -1);
   tocFile.seek(0);
   for (int j = 0; j < tocCount; j++) {
     auto tocEntry = readTocEntry(tocFile);
@@ -185,14 +185,14 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
   // This is O(n*log(m)) instead of O(n*m) while avoiding memory exhaustion.
   // See: https://github.com/crosspoint-reader/crosspoint-reader/issues/134
 
-  std::vector<uint32_t> spineSizes;
+  std::deque<uint32_t> spineSizes;
   bool useBatchSizes = false;
 
   if (spineCount >= LARGE_SPINE_THRESHOLD) {
     LOG_DBG("BMC", "Using batch size lookup for %d spine items", spineCount);
 
-    std::vector<ZipFile::SizeTarget> targets;
-    targets.reserve(spineCount);
+    std::deque<ZipFile::SizeTarget> targets;
+    targets.resize(spineCount);
 
     spineFile.seek(0);
     for (int i = 0; i < spineCount; i++) {
@@ -203,7 +203,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
       t.hash = ZipFile::fnvHash64(path.c_str(), path.size());
       t.len = static_cast<uint16_t>(path.size());
       t.index = static_cast<uint16_t>(i);
-      targets.push_back(t);
+      targets[i] = t;
     }
 
     std::sort(targets.begin(), targets.end(), [](const ZipFile::SizeTarget& a, const ZipFile::SizeTarget& b) {

lib/Epub/Epub/BookMetadataCache.h

@@ -3,8 +3,8 @@
 #include <HalStorage.h>
 
 #include <algorithm>
+#include <deque>
 #include <string>
-#include <vector>
 
 class BookMetadataCache {
  public:
@@ -64,7 +64,7 @@ class BookMetadataCache {
     uint16_t hrefLen;   // length for collision reduction
     int16_t spineIndex;
   };
-  std::vector<SpineHrefIndexEntry> spineHrefIndex;
+  std::deque<SpineHrefIndexEntry> spineHrefIndex;
   bool useSpineHrefIndex = false;
 
   static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400;

lib/Epub/Epub/parsers/ContentOpfParser.h

@@ -2,6 +2,7 @@
 #include <Print.h>
 
 #include <algorithm>
+#include <deque>
 #include <vector>
 
 #include "Epub.h"
@@ -40,7 +41,7 @@ class ContentOpfParser final : public Print {
     uint16_t idLen;       // length for collision reduction
     uint32_t fileOffset;  // offset in .items.bin
   };
-  std::vector<ItemIndexEntry> itemIndex;
+  std::deque<ItemIndexEntry> itemIndex;
   bool useItemIndex = false;
 
   static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400;

lib/ZipFile/ZipFile.cpp

@@ -295,7 +295,7 @@ bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) {
   return true;
 }
 
-int ZipFile::fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes) {
+int ZipFile::fillUncompressedSizes(std::deque<SizeTarget>& targets, std::deque<uint32_t>& sizes) {
   if (targets.empty()) {
     return 0;
   }

lib/ZipFile/ZipFile.h

@@ -1,9 +1,9 @@
 #pragma once
 #include <HalStorage.h>
 
+#include <deque>
 #include <string>
 #include <unordered_map>
-#include <vector>
 
 class ZipFile {
  public:
@@ -64,7 +64,7 @@ class ZipFile {
   // Batch lookup: scan ZIP central dir once and fill sizes for matching targets.
   // targets must be sorted by (hash, len). sizes[target.index] receives uncompressedSize.
   // Returns number of targets matched.
-  int fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes);
+  int fillUncompressedSizes(std::deque<SizeTarget>& targets, std::deque<uint32_t>& sizes);
   // Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple
   // These functions will open and close the zip as needed
   uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);