feat: add png jpeg support (#556)

## Summary - Add embedded image support to EPUB rendering with JPEG and PNG decoders - Implement pixel caching system to cache decoded/dithered images to SD card for faster re-rendering - Add 4-level grayscale support for display ## Changes ### New Image Rendering System - Add `ImageBlock` class to represent an image with its cached path and display dimensions - Add `PageImage` class as a new `PageElement` type for images on pages - Add `ImageToFramebufferDecoder` interface for format-specific image decoders - Add `JpegToFramebufferConverter` - JPEG decoder with Bayer dithering and scaling - Add `PngToFramebufferConverter` - PNG decoder with Bayer dithering and scaling - Add `ImageDecoderFactory` to select appropriate decoder based on file extension - Add `getRenderMode()` to GfxRenderer for grayscale render mode queries ### Dithering and Grayscale - Implement 4x4 Bayer ordered dithering for 4-level grayscale output - Stateless algorithm works correctly with MCU block decoding - Handles scaling without artifacts - Add grayscale render mode support (BW, GRAYSCALE_LSB, GRAYSCALE_MSB) - Image decoders and cache renderer respect current render mode - Enables proper 4-level e-ink grayscale when anti-aliasing is enabled ### Pixel Caching - Cache decoded/dithered images to `.pxc` files on SD card - Cache format: 2-bit packed pixels (4 pixels per byte) with width/height header - On subsequent renders, load directly from cache instead of re-decoding - Cache renderer supports grayscale render modes for multi-pass rendering - Significantly improves page navigation speed for image-heavy EPUBs ### HTML Parser Integration - Update `ChapterHtmlSlimParser` to process `<img>` tags and extract images from EPUB - Resolve relative image paths within EPUB ZIP structure - Extract images to cache directory before decoding - Create `PageImage` elements with proper scaling to fit viewport - Fall back to alt text display if image processing fails ### Build Configuration - Add `PNG_MAX_BUFFERED_PIXELS=6402` to support up to 800px wide images ### Test Script - Generate test EPUBs with annotated JPEG and PNG images - Test cases cover: grayscale (4 levels), centering, scaling, cache performance ## Test plan - [x] Open EPUB with JPEG images - verify images display with proper grayscale - [x] Open EPUB with PNG images - verify images display correctly and no crash - [x] Navigate away from image page and back - verify faster load from cache - [x] Verify grayscale tones render correctly (not just black/white dithering) - [x] Verify large images are scaled down to fit screen - [x] Verify images are centered horizontally - [x] Verify page serialization/deserialization works with images - [x] Verify images rendered in landscape mode ## Test Results [png](https://photos.app.goo.gl/5zFUb8xA8db3dPd19) [jpeg](https://photos.app.goo.gl/SwtwaL2DSQwKybhw7) ![20260128_231123790](https://github.com/user-attachments/assets/78855971-4bb8-441a-b207-0a292b9739f5) ![20260128_231012253](https://github.com/user-attachments/assets/f08fb63f-1b73-41d9-a25e-78232ec0c495) ![20260128_231004209](https://github.com/user-attachments/assets/06c94acc-8a06-4955-978e-6e583399478d) ![20260128_230954997](https://github.com/user-attachments/assets/49bc44d5-0f2c-416b-9199-4d680fb0f4c3) ![20260128_230945717](https://github.com/user-attachments/assets/93446da5-2e07-410c-89c9-6a21d14e5acb) ![20260128_230938313](https://github.com/user-attachments/assets/4c74c72a-3d40-4a25-b0f3-acc703f42c00) ![20260128_230925546](https://github.com/user-attachments/assets/8d8f62ee-c8fc-4f19-a12c-da29083bb766) ![20260128_230918374](https://github.com/user-attachments/assets/f007d5db-41cc-4fa6-bb22-9e767ee7b00d) --- ### AI Usage Did you use AI tools to help write this code? _**< YES >**_ --------- Co-authored-by: Matthías Páll Gissurarson <mpg@mpg.is> Co-authored-by: Dave Allie <dave@daveallie.com>
2026-02-16 19:56:59 +11:00
parent 5ec4210f23
commit ddc31546db
26 changed files with 2060 additions and 32 deletions
@@ -0,0 +1,364 @@
+#include "PngToFramebufferConverter.h"
+
+#include <GfxRenderer.h>
+#include <HardwareSerial.h>
+#include <PNGdec.h>
+#include <SDCardManager.h>
+#include <SdFat.h>
+
+#include <cstdlib>
+#include <new>
+
+#include "DitherUtils.h"
+#include "PixelCache.h"
+
+namespace {
+
+// Context struct passed through PNGdec callbacks to avoid global mutable state.
+// The draw callback receives this via pDraw->pUser (set by png.decode()).
+// The file I/O callbacks receive the FsFile* via pFile->fHandle (set by pngOpen()).
+struct PngContext {
+  GfxRenderer* renderer;
+  const RenderConfig* config;
+  int screenWidth;
+  int screenHeight;
+
+  // Scaling state
+  float scale;
+  int srcWidth;
+  int srcHeight;
+  int dstWidth;
+  int dstHeight;
+  int lastDstY;  // Track last rendered destination Y to avoid duplicates
+
+  PixelCache cache;
+  bool caching;
+
+  uint8_t* grayLineBuffer;
+
+  PngContext()
+      : renderer(nullptr),
+        config(nullptr),
+        screenWidth(0),
+        screenHeight(0),
+        scale(1.0f),
+        srcWidth(0),
+        srcHeight(0),
+        dstWidth(0),
+        dstHeight(0),
+        lastDstY(-1),
+        caching(false),
+        grayLineBuffer(nullptr) {}
+};
+
+// File I/O callbacks use pFile->fHandle to access the FsFile*,
+// avoiding the need for global file state.
+void* pngOpenWithHandle(const char* filename, int32_t* size) {
+  FsFile* f = new FsFile();
+  if (!Storage.openFileForRead("PNG", std::string(filename), *f)) {
+    delete f;
+    return nullptr;
+  }
+  *size = f->size();
+  return f;
+}
+
+void pngCloseWithHandle(void* handle) {
+  FsFile* f = reinterpret_cast<FsFile*>(handle);
+  if (f) {
+    f->close();
+    delete f;
+  }
+}
+
+int32_t pngReadWithHandle(PNGFILE* pFile, uint8_t* pBuf, int32_t len) {
+  FsFile* f = reinterpret_cast<FsFile*>(pFile->fHandle);
+  if (!f) return 0;
+  return f->read(pBuf, len);
+}
+
+int32_t pngSeekWithHandle(PNGFILE* pFile, int32_t pos) {
+  FsFile* f = reinterpret_cast<FsFile*>(pFile->fHandle);
+  if (!f) return -1;
+  return f->seek(pos);
+}
+
+// The PNG decoder (PNGdec) is ~42 KB due to internal zlib decompression buffers.
+// We heap-allocate it on demand rather than using a static instance, so this memory
+// is only consumed while actually decoding/querying PNG images. This is critical on
+// the ESP32-C3 where total RAM is ~320 KB.
+constexpr size_t PNG_DECODER_APPROX_SIZE = 44 * 1024;                          // ~42 KB + overhead
+constexpr size_t MIN_FREE_HEAP_FOR_PNG = PNG_DECODER_APPROX_SIZE + 16 * 1024;  // decoder + 16 KB headroom
+
+// Convert entire source line to grayscale with alpha blending to white background.
+// For indexed PNGs with tRNS chunk, alpha values are stored at palette[768] onwards.
+// Processing the whole line at once improves cache locality and reduces per-pixel overhead.
+void convertLineToGray(uint8_t* pPixels, uint8_t* grayLine, int width, int pixelType, uint8_t* palette, int hasAlpha) {
+  switch (pixelType) {
+    case PNG_PIXEL_GRAYSCALE:
+      memcpy(grayLine, pPixels, width);
+      break;
+
+    case PNG_PIXEL_TRUECOLOR:
+      for (int x = 0; x < width; x++) {
+        uint8_t* p = &pPixels[x * 3];
+        grayLine[x] = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
+      }
+      break;
+
+    case PNG_PIXEL_INDEXED:
+      if (palette) {
+        if (hasAlpha) {
+          for (int x = 0; x < width; x++) {
+            uint8_t idx = pPixels[x];
+            uint8_t* p = &palette[idx * 3];
+            uint8_t gray = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
+            uint8_t alpha = palette[768 + idx];
+            grayLine[x] = (uint8_t)((gray * alpha + 255 * (255 - alpha)) / 255);
+          }
+        } else {
+          for (int x = 0; x < width; x++) {
+            uint8_t* p = &palette[pPixels[x] * 3];
+            grayLine[x] = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
+          }
+        }
+      } else {
+        memcpy(grayLine, pPixels, width);
+      }
+      break;
+
+    case PNG_PIXEL_GRAY_ALPHA:
+      for (int x = 0; x < width; x++) {
+        uint8_t gray = pPixels[x * 2];
+        uint8_t alpha = pPixels[x * 2 + 1];
+        grayLine[x] = (uint8_t)((gray * alpha + 255 * (255 - alpha)) / 255);
+      }
+      break;
+
+    case PNG_PIXEL_TRUECOLOR_ALPHA:
+      for (int x = 0; x < width; x++) {
+        uint8_t* p = &pPixels[x * 4];
+        uint8_t gray = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
+        uint8_t alpha = p[3];
+        grayLine[x] = (uint8_t)((gray * alpha + 255 * (255 - alpha)) / 255);
+      }
+      break;
+
+    default:
+      memset(grayLine, 128, width);
+      break;
+  }
+}
+
+int pngDrawCallback(PNGDRAW* pDraw) {
+  PngContext* ctx = reinterpret_cast<PngContext*>(pDraw->pUser);
+  if (!ctx || !ctx->config || !ctx->renderer || !ctx->grayLineBuffer) return 0;
+
+  int srcY = pDraw->y;
+  int srcWidth = ctx->srcWidth;
+
+  // Calculate destination Y with scaling
+  int dstY = (int)(srcY * ctx->scale);
+
+  // Skip if we already rendered this destination row (multiple source rows map to same dest)
+  if (dstY == ctx->lastDstY) return 1;
+  ctx->lastDstY = dstY;
+
+  // Check bounds
+  if (dstY >= ctx->dstHeight) return 1;
+
+  int outY = ctx->config->y + dstY;
+  if (outY >= ctx->screenHeight) return 1;
+
+  // Convert entire source line to grayscale (improves cache locality)
+  convertLineToGray(pDraw->pPixels, ctx->grayLineBuffer, srcWidth, pDraw->iPixelType, pDraw->pPalette,
+                    pDraw->iHasAlpha);
+
+  // Render scaled row using Bresenham-style integer stepping (no floating-point division)
+  int dstWidth = ctx->dstWidth;
+  int outXBase = ctx->config->x;
+  int screenWidth = ctx->screenWidth;
+  bool useDithering = ctx->config->useDithering;
+  bool caching = ctx->caching;
+
+  int srcX = 0;
+  int error = 0;
+
+  for (int dstX = 0; dstX < dstWidth; dstX++) {
+    int outX = outXBase + dstX;
+    if (outX < screenWidth) {
+      uint8_t gray = ctx->grayLineBuffer[srcX];
+
+      uint8_t ditheredGray;
+      if (useDithering) {
+        ditheredGray = applyBayerDither4Level(gray, outX, outY);
+      } else {
+        ditheredGray = gray / 85;
+        if (ditheredGray > 3) ditheredGray = 3;
+      }
+      drawPixelWithRenderMode(*ctx->renderer, outX, outY, ditheredGray);
+      if (caching) ctx->cache.setPixel(outX, outY, ditheredGray);
+    }
+
+    // Bresenham-style stepping: advance srcX based on ratio srcWidth/dstWidth
+    error += srcWidth;
+    while (error >= dstWidth) {
+      error -= dstWidth;
+      srcX++;
+    }
+  }
+
+  return 1;
+}
+
+}  // namespace
+
+bool PngToFramebufferConverter::getDimensionsStatic(const std::string& imagePath, ImageDimensions& out) {
+  size_t freeHeap = ESP.getFreeHeap();
+  if (freeHeap < MIN_FREE_HEAP_FOR_PNG) {
+    Serial.printf("[%lu] [PNG] Not enough heap for PNG decoder (%u free, need %u)\n", millis(), freeHeap,
+                  MIN_FREE_HEAP_FOR_PNG);
+    return false;
+  }
+
+  PNG* png = new (std::nothrow) PNG();
+  if (!png) {
+    Serial.printf("[%lu] [PNG] Failed to allocate PNG decoder for dimensions\n", millis());
+    return false;
+  }
+
+  int rc = png->open(imagePath.c_str(), pngOpenWithHandle, pngCloseWithHandle, pngReadWithHandle, pngSeekWithHandle,
+                     nullptr);
+
+  if (rc != 0) {
+    Serial.printf("[%lu] [PNG] Failed to open PNG for dimensions: %d\n", millis(), rc);
+    delete png;
+    return false;
+  }
+
+  out.width = png->getWidth();
+  out.height = png->getHeight();
+
+  png->close();
+  delete png;
+  return true;
+}
+
+bool PngToFramebufferConverter::decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer,
+                                                    const RenderConfig& config) {
+  Serial.printf("[%lu] [PNG] Decoding PNG: %s\n", millis(), imagePath.c_str());
+
+  size_t freeHeap = ESP.getFreeHeap();
+  if (freeHeap < MIN_FREE_HEAP_FOR_PNG) {
+    Serial.printf("[%lu] [PNG] Not enough heap for PNG decoder (%u free, need %u)\n", millis(), freeHeap,
+                  MIN_FREE_HEAP_FOR_PNG);
+    return false;
+  }
+
+  // Heap-allocate PNG decoder (~42 KB) - freed at end of function
+  PNG* png = new (std::nothrow) PNG();
+  if (!png) {
+    Serial.printf("[%lu] [PNG] Failed to allocate PNG decoder\n", millis());
+    return false;
+  }
+
+  PngContext ctx;
+  ctx.renderer = &renderer;
+  ctx.config = &config;
+  ctx.screenWidth = renderer.getScreenWidth();
+  ctx.screenHeight = renderer.getScreenHeight();
+
+  int rc = png->open(imagePath.c_str(), pngOpenWithHandle, pngCloseWithHandle, pngReadWithHandle, pngSeekWithHandle,
+                     pngDrawCallback);
+  if (rc != PNG_SUCCESS) {
+    Serial.printf("[%lu] [PNG] Failed to open PNG: %d\n", millis(), rc);
+    delete png;
+    return false;
+  }
+
+  if (!validateImageDimensions(png->getWidth(), png->getHeight(), "PNG")) {
+    png->close();
+    delete png;
+    return false;
+  }
+
+  // Calculate output dimensions
+  ctx.srcWidth = png->getWidth();
+  ctx.srcHeight = png->getHeight();
+
+  if (config.useExactDimensions && config.maxWidth > 0 && config.maxHeight > 0) {
+    // Use exact dimensions as specified (avoids rounding mismatches with pre-calculated sizes)
+    ctx.dstWidth = config.maxWidth;
+    ctx.dstHeight = config.maxHeight;
+    ctx.scale = (float)ctx.dstWidth / ctx.srcWidth;
+  } else {
+    // Calculate scale factor to fit within maxWidth/maxHeight
+    float scaleX = (float)config.maxWidth / ctx.srcWidth;
+    float scaleY = (float)config.maxHeight / ctx.srcHeight;
+    ctx.scale = (scaleX < scaleY) ? scaleX : scaleY;
+    if (ctx.scale > 1.0f) ctx.scale = 1.0f;  // Don't upscale
+
+    ctx.dstWidth = (int)(ctx.srcWidth * ctx.scale);
+    ctx.dstHeight = (int)(ctx.srcHeight * ctx.scale);
+  }
+  ctx.lastDstY = -1;  // Reset row tracking
+
+  Serial.printf("[%lu] [PNG] PNG %dx%d -> %dx%d (scale %.2f), bpp: %d\n", millis(), ctx.srcWidth, ctx.srcHeight,
+                ctx.dstWidth, ctx.dstHeight, ctx.scale, png->getBpp());
+
+  if (png->getBpp() != 8) {
+    warnUnsupportedFeature("bit depth (" + std::to_string(png->getBpp()) + "bpp)", imagePath);
+  }
+
+  // Allocate grayscale line buffer on demand (~3.2 KB) - freed after decode
+  const size_t grayBufSize = PNG_MAX_BUFFERED_PIXELS / 2;
+  ctx.grayLineBuffer = static_cast<uint8_t*>(malloc(grayBufSize));
+  if (!ctx.grayLineBuffer) {
+    Serial.printf("[%lu] [PNG] Failed to allocate gray line buffer\n", millis());
+    png->close();
+    delete png;
+    return false;
+  }
+
+  // Allocate cache buffer using SCALED dimensions
+  ctx.caching = !config.cachePath.empty();
+  if (ctx.caching) {
+    if (!ctx.cache.allocate(ctx.dstWidth, ctx.dstHeight, config.x, config.y)) {
+      Serial.printf("[%lu] [PNG] Failed to allocate cache buffer, continuing without caching\n", millis());
+      ctx.caching = false;
+    }
+  }
+
+  unsigned long decodeStart = millis();
+  rc = png->decode(&ctx, 0);
+  unsigned long decodeTime = millis() - decodeStart;
+
+  free(ctx.grayLineBuffer);
+  ctx.grayLineBuffer = nullptr;
+
+  if (rc != PNG_SUCCESS) {
+    Serial.printf("[%lu] [PNG] Decode failed: %d\n", millis(), rc);
+    png->close();
+    delete png;
+    return false;
+  }
+
+  png->close();
+  delete png;
+  Serial.printf("[%lu] [PNG] PNG decoding complete - render time: %lu ms\n", millis(), decodeTime);
+
+  // Write cache file if caching was enabled and buffer was allocated
+  if (ctx.caching) {
+    ctx.cache.writeToFile(config.cachePath);
+  }
+
+  return true;
+}
+
+bool PngToFramebufferConverter::supportsFormat(const std::string& extension) {
+  std::string ext = extension;
+  for (auto& c : ext) {
+    c = tolower(c);
+  }
+  return (ext == ".png");
+}