fix: Add framebuffer release/realloc and improved lazy indexing (#2563)

lib/miniz/library.json

@@ -0,0 +1,9 @@
+{
+  "name": "miniz",
+  "version": "11.3.2",
+  "description": "Vendored miniz (tinfl inflate only) + InflateStream wrapper",
+  "build": {
+    "srcDir": "src",
+    "includeDir": "src"
+  }
+}

lib/miniz/src/InflateStream.cpp

@@ -0,0 +1,165 @@
+#include "InflateStream.h"
+
+#include <BuildScratch.h>
+
+#include <cstdlib>
+#include <cstring>
+
+#include "MinizConfig.h"
+
+namespace {
+// tinfl's window must be a power of two; TINFL_LZ_DICT_SIZE is 32768.
+constexpr size_t WINDOW_SIZE = TINFL_LZ_DICT_SIZE;
+// tinfl_decompressor holds mz_uint32 arrays; 8 keeps the window aligned too.
+constexpr size_t STATE_ALIGNED = (sizeof(tinfl_decompressor) + 7) & ~size_t{7};
+}  // namespace
+
+InflateStream::~InflateStream() { deinit(); }
+
+bool InflateStream::init(const bool streaming) {
+  // Every consumer constructs a fresh stream per operation, so acquire storage
+  // from scratch each init (releasing any prior backing first).
+  deinit();
+
+  // During a framebuffer loan the lent 48KB is up for grabs: state (~11KB) +
+  // window (32KB) fit inside it, so a chapter-build inflate costs the heap
+  // nothing. Absent (or already claimed): plain heap, freed in deinit().
+  const size_t needed = STATE_ALIGNED + (streaming ? WINDOW_SIZE : 0);
+  arenaBase = buildscratch::claim(needed);
+  if (arenaBase) {
+    state = reinterpret_cast<tinfl_decompressor*>(arenaBase);
+    window = streaming ? arenaBase + STATE_ALIGNED : nullptr;
+  } else {
+    // Raw malloc (not makeUniqueNoThrow): the header keeps tinfl_decompressor
+    // an incomplete type so consumers never include miniz; both blocks are
+    // freed in deinit()/the destructor.
+    state = static_cast<tinfl_decompressor*>(malloc(sizeof(tinfl_decompressor)));
+    if (!state) return false;
+    if (streaming) {
+      window = static_cast<uint8_t*>(malloc(WINDOW_SIZE));
+      if (!window) return false;  // state kept; deinit()/next init reclaims it
+    }
+  }
+
+  tinfl_init(state);
+  windowPos = 0;
+  pendingStart = 0;
+  pendingLen = 0;
+  inPtr = nullptr;
+  inAvail = 0;
+  fill = nullptr;
+  fillCtx = nullptr;
+  inputExhausted = false;
+  zlibWrapped = false;
+  finished = false;
+  oneShotStart = nullptr;
+  return true;
+}
+
+void InflateStream::deinit() {
+  if (arenaBase) {
+    buildscratch::release(arenaBase);
+    arenaBase = nullptr;
+  } else {
+    free(state);
+    free(window);
+  }
+  state = nullptr;
+  window = nullptr;
+}
+
+void InflateStream::setSource(const uint8_t* src, const size_t len) {
+  inPtr = src;
+  inAvail = len;
+  inputExhausted = true;  // the whole input is present; nothing more will come
+}
+
+void InflateStream::setFill(const FillFn fn, void* ctx) {
+  fill = fn;
+  fillCtx = ctx;
+}
+
+InflateStream::Status InflateStream::readAtMost(uint8_t* dest, const size_t maxLen, size_t* produced) {
+  *produced = 0;
+  if (!state) return Status::Error;
+
+  const bool streaming = window != nullptr;
+  if (!streaming && !oneShotStart) oneShotStart = dest;
+
+  for (;;) {
+    // Drain window bytes left over from a previous tinfl call. In ring mode
+    // tinfl may produce more than the caller asked for in one shot -- the
+    // overshoot stays pending in the window until a later readAtMost.
+    if (pendingLen > 0) {
+      size_t n = maxLen - *produced;
+      if (n > pendingLen) n = pendingLen;
+      memcpy(dest + *produced, window + pendingStart, n);
+      pendingStart += n;
+      pendingLen -= n;
+      *produced += n;
+    }
+    if (*produced == maxLen) {
+      return (finished && pendingLen == 0) ? Status::Done : Status::Ok;
+    }
+    if (finished) return Status::Done;
+
+    if (inAvail == 0 && !inputExhausted && fill) {
+      inAvail = fill(fillCtx, &inPtr);
+      if (inAvail == 0) inputExhausted = true;
+    }
+
+    const mz_uint32 flags = (zlibWrapped ? TINFL_FLAG_PARSE_ZLIB_HEADER : 0) |
+                            (inputExhausted ? 0 : TINFL_FLAG_HAS_MORE_INPUT) |
+                            (streaming ? 0 : TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);
+
+    size_t inBytes = inAvail;
+    tinfl_status status;
+    size_t outBytes;
+    if (streaming) {
+      // Ring mode invariant: tinfl derives its wrap mask from
+      // (cursor offset + avail_out), so avail_out MUST always reach the end of
+      // the 32KB window -- never cap it to the caller's remaining space.
+      outBytes = WINDOW_SIZE - windowPos;
+      status = tinfl_decompress(state, inPtr, &inBytes, window, window + windowPos, &outBytes, flags);
+      pendingStart = windowPos;
+      pendingLen = outBytes;
+      windowPos += outBytes;
+      if (windowPos == WINDOW_SIZE) windowPos = 0;
+    } else {
+      // One-shot: back-references resolve directly inside the destination buffer.
+      outBytes = maxLen - *produced;
+      status = tinfl_decompress(state, inPtr, &inBytes, oneShotStart, dest + *produced, &outBytes, flags);
+      *produced += outBytes;
+    }
+    inPtr += inBytes;
+    inAvail -= inBytes;
+
+    if (status == TINFL_STATUS_DONE) {
+      finished = true;  // drain any pending window bytes on the next pass
+      continue;
+    }
+    if (status < TINFL_STATUS_DONE) return Status::Error;  // corrupt stream / adler mismatch
+    // TINFL_STATUS_NEEDS_MORE_INPUT loops back to the fill above; once the fill
+    // runs dry the HAS_MORE_INPUT flag drops and tinfl either finishes or fails
+    // (truncated stream) instead of spinning.
+    if (status == TINFL_STATUS_NEEDS_MORE_INPUT && inputExhausted && inAvail == 0) {
+      return Status::Error;
+    }
+    if (*produced == maxLen) {
+      return (finished && pendingLen == 0) ? Status::Done : Status::Ok;
+    }
+  }
+}
+
+bool InflateStream::read(uint8_t* dest, const size_t len) {
+  size_t total = 0;
+  while (total < len) {
+    size_t produced = 0;
+    const Status status = readAtMost(dest + total, len - total, &produced);
+    total += produced;
+    if (status == Status::Error) return false;
+    if (status == Status::Done) return total == len;
+    if (produced == 0) return false;  // no progress safeguard
+  }
+  return true;
+}

lib/miniz/src/InflateStream.h

@@ -0,0 +1,95 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// Forward declaration keeps miniz out of consumer translation units; the
+// decompressor state is heap-allocated in the .cpp where the type is complete.
+struct tinfl_decompressor_tag;
+
+// Streaming deflate decompressor wrapping miniz's tinfl.
+//
+// Replaces the uzlib-backed InflateReader on the throughput paths (EPUB zip
+// entries, PNG IDAT). tinfl decodes via lookup tables where uzlib walks the
+// Huffman tree bit-by-bit -- several times faster on this CPU -- at the cost
+// of a larger decompressor state (~11KB, transient for the scope of the
+// stream; taken from the lent framebuffer bytes via buildscratch::claim()
+// when a FrameBufferLoan is active, heap otherwise). FontDecompressor
+// intentionally stays on InflateReader:
+// its one-shot flash-resident group decompressions are tiny, and the render
+// path should not carry the extra state allocation.
+//
+// Two modes:
+//   init(false) -- one-shot: the destination buffer holds the ENTIRE output,
+//                  so back-references resolve inside it and no 32KB window is
+//                  allocated. read()/readAtMost() must be driven with
+//                  contiguous, forward-only slices of that one buffer
+//                  (a single read(dest, totalSize) is the common case).
+//   init(true)  -- streaming: allocates a 32KB window; output can go to any
+//                  buffer in any-sized chunks across calls.
+//
+// Input is either a single contiguous buffer (setSource) or pulled on demand
+// through a fill callback (setFill): return the number of bytes available and
+// point *data at them (valid until the next fill call); return 0 at end of
+// input. Call setZlibWrapped() before the first read when the stream has a
+// zlib header (e.g. PNG IDAT).
+class InflateStream {
+ public:
+  enum class Status {
+    Ok,     // Output buffer full; more decompressed data remains.
+    Done,   // Stream ended cleanly. produced may be < maxLen.
+    Error,  // Corrupt/truncated stream, or decompression failed.
+  };
+
+  using FillFn = size_t (*)(void* ctx, const uint8_t** data);
+
+  InflateStream() = default;
+  ~InflateStream();
+  InflateStream(const InflateStream&) = delete;
+  InflateStream& operator=(const InflateStream&) = delete;
+
+  // Allocate decompressor state (and the 32KB window when streaming) and reset
+  // stream state. Reuses existing allocations on repeated calls. Returns false
+  // on OOM.
+  bool init(bool streaming);
+
+  // Free the decompressor state and window.
+  void deinit();
+
+  // Provide the entire compressed input as one contiguous buffer.
+  void setSource(const uint8_t* src, size_t len);
+
+  // Provide compressed input on demand. ctx is passed back to fn verbatim.
+  void setFill(FillFn fn, void* ctx);
+
+  // Declare the input zlib-wrapped (2-byte header + trailing adler32).
+  void setZlibWrapped() { zlibWrapped = true; }
+
+  // Decompress exactly len bytes into dest. Returns false if the stream ends
+  // or errors before producing len bytes.
+  bool read(uint8_t* dest, size_t len);
+
+  // Decompress up to maxLen bytes into dest; *produced gets the byte count.
+  Status readAtMost(uint8_t* dest, size_t maxLen, size_t* produced);
+
+ private:
+  tinfl_decompressor_tag* state = nullptr;  // ~11KB: heap, or inside the claimed build scratch
+  uint8_t* window = nullptr;                // 32KB ring, streaming mode only
+  uint8_t* arenaBase = nullptr;             // non-null when state/window live in lent framebuffer bytes
+  size_t windowPos = 0;                     // ring write cursor
+  // Decompressed-but-undelivered region of the window (tinfl can overshoot the
+  // caller's requested length; the overshoot waits here for the next read).
+  size_t pendingStart = 0;
+  size_t pendingLen = 0;
+
+  const uint8_t* inPtr = nullptr;
+  size_t inAvail = 0;
+  FillFn fill = nullptr;
+  void* fillCtx = nullptr;
+  bool inputExhausted = false;
+  bool zlibWrapped = false;
+  bool finished = false;
+
+  // One-shot mode: tinfl needs the output buffer start for back-references.
+  uint8_t* oneShotStart = nullptr;
+};

lib/miniz/src/MinizConfig.h

@@ -0,0 +1,35 @@
+/* CrossPoint only needs miniz's low-level streaming inflate (tinfl). The
+ * archive, deflate, stdio, and zlib-compatibility layers are compiled out so
+ * the vendored library stays small and never touches the filesystem or clock.
+ * Include this header instead of <miniz.h> so every translation unit sees the
+ * same configuration. */
+#pragma once
+
+#define MINIZ_NO_STDIO
+#define MINIZ_NO_TIME
+#define MINIZ_NO_ARCHIVE_APIS
+#define MINIZ_NO_ARCHIVE_WRITING_APIS
+#define MINIZ_NO_DEFLATE_APIS
+#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES
+
+// The ESP32 mask ROM exports tinfl_* at fixed addresses via DIRECT linker
+// script assignments (e.g. "tinfl_decompress = 0x...;" in the ROM .ld),
+// which override object-file definitions -- without these renames the
+// firmware silently binds to the ROM's 2021 build (TINFL_LESS_MEMORY, a
+// different tinfl_decompressor layout) and corrupts inflate state on real
+// data. Rename so the linker can never capture them. The prefix is
+// crosspoint_ (NOT freeink_) so a future branch that links FreeInkBook's
+// identically-renamed copy does not collide.
+#define tinfl_decompress crosspoint_tinfl_decompress
+#define tinfl_decompress_mem_to_heap crosspoint_tinfl_decompress_mem_to_heap
+#define tinfl_decompress_mem_to_mem crosspoint_tinfl_decompress_mem_to_mem
+#define tinfl_decompress_mem_to_callback crosspoint_tinfl_decompress_mem_to_callback
+#define mz_crc32 crosspoint_mz_crc32
+#define mz_adler32 crosspoint_mz_adler32
+#define mz_free crosspoint_mz_free
+
+// Include the vendored miniz by relative path: ESP-IDF ships a ROM miniz.h
+// with the SAME include guard but a different (TINFL_LESS_MEMORY) struct
+// layout -- resolving <miniz.h> through the platform include path would
+// silently compile against the wrong structures.
+#include "../third_party/miniz.h"

lib/miniz/src/miniz_impl.c

@@ -0,0 +1,7 @@
+/* Compiles the vendored miniz with CrossPoint's configuration. The include
+ * order is load-bearing (the config defines/renames must be seen first). */
+// clang-format off
+#include "MinizConfig.h"
+
+#include "../third_party/miniz.c"
+// clang-format on