feat: add png jpeg support (#556)

## Summary - Add embedded image support to EPUB rendering with JPEG and PNG decoders - Implement pixel caching system to cache decoded/dithered images to SD card for faster re-rendering - Add 4-level grayscale support for display ## Changes ### New Image Rendering System - Add `ImageBlock` class to represent an image with its cached path and display dimensions - Add `PageImage` class as a new `PageElement` type for images on pages - Add `ImageToFramebufferDecoder` interface for format-specific image decoders - Add `JpegToFramebufferConverter` - JPEG decoder with Bayer dithering and scaling - Add `PngToFramebufferConverter` - PNG decoder with Bayer dithering and scaling - Add `ImageDecoderFactory` to select appropriate decoder based on file extension - Add `getRenderMode()` to GfxRenderer for grayscale render mode queries ### Dithering and Grayscale - Implement 4x4 Bayer ordered dithering for 4-level grayscale output - Stateless algorithm works correctly with MCU block decoding - Handles scaling without artifacts - Add grayscale render mode support (BW, GRAYSCALE_LSB, GRAYSCALE_MSB) - Image decoders and cache renderer respect current render mode - Enables proper 4-level e-ink grayscale when anti-aliasing is enabled ### Pixel Caching - Cache decoded/dithered images to `.pxc` files on SD card - Cache format: 2-bit packed pixels (4 pixels per byte) with width/height header - On subsequent renders, load directly from cache instead of re-decoding - Cache renderer supports grayscale render modes for multi-pass rendering - Significantly improves page navigation speed for image-heavy EPUBs ### HTML Parser Integration - Update `ChapterHtmlSlimParser` to process `<img>` tags and extract images from EPUB - Resolve relative image paths within EPUB ZIP structure - Extract images to cache directory before decoding - Create `PageImage` elements with proper scaling to fit viewport - Fall back to alt text display if image processing fails ### Build Configuration - Add `PNG_MAX_BUFFERED_PIXELS=6402` to support up to 800px wide images ### Test Script - Generate test EPUBs with annotated JPEG and PNG images - Test cases cover: grayscale (4 levels), centering, scaling, cache performance ## Test plan - [x] Open EPUB with JPEG images - verify images display with proper grayscale - [x] Open EPUB with PNG images - verify images display correctly and no crash - [x] Navigate away from image page and back - verify faster load from cache - [x] Verify grayscale tones render correctly (not just black/white dithering) - [x] Verify large images are scaled down to fit screen - [x] Verify images are centered horizontally - [x] Verify page serialization/deserialization works with images - [x] Verify images rendered in landscape mode ## Test Results [png](https://photos.app.goo.gl/5zFUb8xA8db3dPd19) [jpeg](https://photos.app.goo.gl/SwtwaL2DSQwKybhw7) ![20260128_231123790](https://github.com/user-attachments/assets/78855971-4bb8-441a-b207-0a292b9739f5) ![20260128_231012253](https://github.com/user-attachments/assets/f08fb63f-1b73-41d9-a25e-78232ec0c495) ![20260128_231004209](https://github.com/user-attachments/assets/06c94acc-8a06-4955-978e-6e583399478d) ![20260128_230954997](https://github.com/user-attachments/assets/49bc44d5-0f2c-416b-9199-4d680fb0f4c3) ![20260128_230945717](https://github.com/user-attachments/assets/93446da5-2e07-410c-89c9-6a21d14e5acb) ![20260128_230938313](https://github.com/user-attachments/assets/4c74c72a-3d40-4a25-b0f3-acc703f42c00) ![20260128_230925546](https://github.com/user-attachments/assets/8d8f62ee-c8fc-4f19-a12c-da29083bb766) ![20260128_230918374](https://github.com/user-attachments/assets/f007d5db-41cc-4fa6-bb22-9e767ee7b00d) --- ### AI Usage Did you use AI tools to help write this code? _**< YES >**_ --------- Co-authored-by: Matthías Páll Gissurarson <mpg@mpg.is> Co-authored-by: Dave Allie <dave@daveallie.com>
2026-02-16 19:56:59 +11:00
parent 5ec4210f23
commit ddc31546db
26 changed files with 2060 additions and 32 deletions
@@ -0,0 +1,298 @@
+#include "JpegToFramebufferConverter.h"
+
+#include <GfxRenderer.h>
+#include <HardwareSerial.h>
+#include <SDCardManager.h>
+#include <SdFat.h>
+#include <picojpeg.h>
+
+#include <cstdio>
+#include <cstring>
+
+#include "DitherUtils.h"
+#include "PixelCache.h"
+
+struct JpegContext {
+  FsFile& file;
+  uint8_t buffer[512];
+  size_t bufferPos;
+  size_t bufferFilled;
+  JpegContext(FsFile& f) : file(f), bufferPos(0), bufferFilled(0) {}
+};
+
+bool JpegToFramebufferConverter::getDimensionsStatic(const std::string& imagePath, ImageDimensions& out) {
+  FsFile file;
+  if (!Storage.openFileForRead("JPG", imagePath, file)) {
+    Serial.printf("[%lu] [JPG] Failed to open file for dimensions: %s\n", millis(), imagePath.c_str());
+    return false;
+  }
+
+  JpegContext context(file);
+  pjpeg_image_info_t imageInfo;
+
+  int status = pjpeg_decode_init(&imageInfo, jpegReadCallback, &context, 0);
+  file.close();
+
+  if (status != 0) {
+    Serial.printf("[%lu] [JPG] Failed to init JPEG for dimensions: %d\n", millis(), status);
+    return false;
+  }
+
+  out.width = imageInfo.m_width;
+  out.height = imageInfo.m_height;
+  Serial.printf("[%lu] [JPG] Image dimensions: %dx%d\n", millis(), out.width, out.height);
+  return true;
+}
+
+bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer,
+                                                     const RenderConfig& config) {
+  Serial.printf("[%lu] [JPG] Decoding JPEG: %s\n", millis(), imagePath.c_str());
+
+  FsFile file;
+  if (!Storage.openFileForRead("JPG", imagePath, file)) {
+    Serial.printf("[%lu] [JPG] Failed to open file: %s\n", millis(), imagePath.c_str());
+    return false;
+  }
+
+  JpegContext context(file);
+  pjpeg_image_info_t imageInfo;
+
+  int status = pjpeg_decode_init(&imageInfo, jpegReadCallback, &context, 0);
+  if (status != 0) {
+    Serial.printf("[%lu] [JPG] picojpeg init failed: %d\n", millis(), status);
+    file.close();
+    return false;
+  }
+
+  if (!validateImageDimensions(imageInfo.m_width, imageInfo.m_height, "JPEG")) {
+    file.close();
+    return false;
+  }
+
+  // Calculate output dimensions
+  int destWidth, destHeight;
+  float scale;
+
+  if (config.useExactDimensions && config.maxWidth > 0 && config.maxHeight > 0) {
+    // Use exact dimensions as specified (avoids rounding mismatches with pre-calculated sizes)
+    destWidth = config.maxWidth;
+    destHeight = config.maxHeight;
+    scale = (float)destWidth / imageInfo.m_width;
+  } else {
+    // Calculate scale factor to fit within maxWidth/maxHeight
+    float scaleX = (config.maxWidth > 0 && imageInfo.m_width > config.maxWidth)
+                       ? (float)config.maxWidth / imageInfo.m_width
+                       : 1.0f;
+    float scaleY = (config.maxHeight > 0 && imageInfo.m_height > config.maxHeight)
+                       ? (float)config.maxHeight / imageInfo.m_height
+                       : 1.0f;
+    scale = (scaleX < scaleY) ? scaleX : scaleY;
+    if (scale > 1.0f) scale = 1.0f;
+
+    destWidth = (int)(imageInfo.m_width * scale);
+    destHeight = (int)(imageInfo.m_height * scale);
+  }
+
+  Serial.printf("[%lu] [JPG] JPEG %dx%d -> %dx%d (scale %.2f), scan type: %d, MCU: %dx%d\n", millis(),
+                imageInfo.m_width, imageInfo.m_height, destWidth, destHeight, scale, imageInfo.m_scanType,
+                imageInfo.m_MCUWidth, imageInfo.m_MCUHeight);
+
+  if (!imageInfo.m_pMCUBufR || !imageInfo.m_pMCUBufG || !imageInfo.m_pMCUBufB) {
+    Serial.printf("[%lu] [JPG] Null buffer pointers in imageInfo\n", millis());
+    file.close();
+    return false;
+  }
+
+  const int screenWidth = renderer.getScreenWidth();
+  const int screenHeight = renderer.getScreenHeight();
+
+  // Allocate pixel cache if cachePath is provided
+  PixelCache cache;
+  bool caching = !config.cachePath.empty();
+  if (caching) {
+    if (!cache.allocate(destWidth, destHeight, config.x, config.y)) {
+      Serial.printf("[%lu] [JPG] Failed to allocate cache buffer, continuing without caching\n", millis());
+      caching = false;
+    }
+  }
+
+  int mcuX = 0;
+  int mcuY = 0;
+
+  while (mcuY < imageInfo.m_MCUSPerCol) {
+    status = pjpeg_decode_mcu();
+    if (status == PJPG_NO_MORE_BLOCKS) {
+      break;
+    }
+    if (status != 0) {
+      Serial.printf("[%lu] [JPG] MCU decode failed: %d\n", millis(), status);
+      file.close();
+      return false;
+    }
+
+    // Source position in image coordinates
+    int srcStartX = mcuX * imageInfo.m_MCUWidth;
+    int srcStartY = mcuY * imageInfo.m_MCUHeight;
+
+    switch (imageInfo.m_scanType) {
+      case PJPG_GRAYSCALE:
+        for (int row = 0; row < 8; row++) {
+          int srcY = srcStartY + row;
+          int destY = config.y + (int)(srcY * scale);
+          if (destY >= screenHeight || destY >= config.y + destHeight) continue;
+          for (int col = 0; col < 8; col++) {
+            int srcX = srcStartX + col;
+            int destX = config.x + (int)(srcX * scale);
+            if (destX >= screenWidth || destX >= config.x + destWidth) continue;
+            uint8_t gray = imageInfo.m_pMCUBufR[row * 8 + col];
+            uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
+            if (dithered > 3) dithered = 3;
+            drawPixelWithRenderMode(renderer, destX, destY, dithered);
+            if (caching) cache.setPixel(destX, destY, dithered);
+          }
+        }
+        break;
+
+      case PJPG_YH1V1:
+        for (int row = 0; row < 8; row++) {
+          int srcY = srcStartY + row;
+          int destY = config.y + (int)(srcY * scale);
+          if (destY >= screenHeight || destY >= config.y + destHeight) continue;
+          for (int col = 0; col < 8; col++) {
+            int srcX = srcStartX + col;
+            int destX = config.x + (int)(srcX * scale);
+            if (destX >= screenWidth || destX >= config.x + destWidth) continue;
+            uint8_t r = imageInfo.m_pMCUBufR[row * 8 + col];
+            uint8_t g = imageInfo.m_pMCUBufG[row * 8 + col];
+            uint8_t b = imageInfo.m_pMCUBufB[row * 8 + col];
+            uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
+            uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
+            if (dithered > 3) dithered = 3;
+            drawPixelWithRenderMode(renderer, destX, destY, dithered);
+            if (caching) cache.setPixel(destX, destY, dithered);
+          }
+        }
+        break;
+
+      case PJPG_YH2V1:
+        for (int row = 0; row < 8; row++) {
+          int srcY = srcStartY + row;
+          int destY = config.y + (int)(srcY * scale);
+          if (destY >= screenHeight || destY >= config.y + destHeight) continue;
+          for (int col = 0; col < 16; col++) {
+            int srcX = srcStartX + col;
+            int destX = config.x + (int)(srcX * scale);
+            if (destX >= screenWidth || destX >= config.x + destWidth) continue;
+            int blockIndex = (col < 8) ? 0 : 1;
+            int pixelIndex = row * 8 + (col % 8);
+            uint8_t r = imageInfo.m_pMCUBufR[blockIndex * 64 + pixelIndex];
+            uint8_t g = imageInfo.m_pMCUBufG[blockIndex * 64 + pixelIndex];
+            uint8_t b = imageInfo.m_pMCUBufB[blockIndex * 64 + pixelIndex];
+            uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
+            uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
+            if (dithered > 3) dithered = 3;
+            drawPixelWithRenderMode(renderer, destX, destY, dithered);
+            if (caching) cache.setPixel(destX, destY, dithered);
+          }
+        }
+        break;
+
+      case PJPG_YH1V2:
+        for (int row = 0; row < 16; row++) {
+          int srcY = srcStartY + row;
+          int destY = config.y + (int)(srcY * scale);
+          if (destY >= screenHeight || destY >= config.y + destHeight) continue;
+          for (int col = 0; col < 8; col++) {
+            int srcX = srcStartX + col;
+            int destX = config.x + (int)(srcX * scale);
+            if (destX >= screenWidth || destX >= config.x + destWidth) continue;
+            int blockIndex = (row < 8) ? 0 : 1;
+            int pixelIndex = (row % 8) * 8 + col;
+            uint8_t r = imageInfo.m_pMCUBufR[blockIndex * 128 + pixelIndex];
+            uint8_t g = imageInfo.m_pMCUBufG[blockIndex * 128 + pixelIndex];
+            uint8_t b = imageInfo.m_pMCUBufB[blockIndex * 128 + pixelIndex];
+            uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
+            uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
+            if (dithered > 3) dithered = 3;
+            drawPixelWithRenderMode(renderer, destX, destY, dithered);
+            if (caching) cache.setPixel(destX, destY, dithered);
+          }
+        }
+        break;
+
+      case PJPG_YH2V2:
+        for (int row = 0; row < 16; row++) {
+          int srcY = srcStartY + row;
+          int destY = config.y + (int)(srcY * scale);
+          if (destY >= screenHeight || destY >= config.y + destHeight) continue;
+          for (int col = 0; col < 16; col++) {
+            int srcX = srcStartX + col;
+            int destX = config.x + (int)(srcX * scale);
+            if (destX >= screenWidth || destX >= config.x + destWidth) continue;
+            int blockX = (col < 8) ? 0 : 1;
+            int blockY = (row < 8) ? 0 : 1;
+            int blockIndex = blockY * 2 + blockX;
+            int pixelIndex = (row % 8) * 8 + (col % 8);
+            int blockOffset = blockIndex * 64;
+            uint8_t r = imageInfo.m_pMCUBufR[blockOffset + pixelIndex];
+            uint8_t g = imageInfo.m_pMCUBufG[blockOffset + pixelIndex];
+            uint8_t b = imageInfo.m_pMCUBufB[blockOffset + pixelIndex];
+            uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
+            uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
+            if (dithered > 3) dithered = 3;
+            drawPixelWithRenderMode(renderer, destX, destY, dithered);
+            if (caching) cache.setPixel(destX, destY, dithered);
+          }
+        }
+        break;
+    }
+
+    mcuX++;
+    if (mcuX >= imageInfo.m_MCUSPerRow) {
+      mcuX = 0;
+      mcuY++;
+    }
+  }
+
+  Serial.printf("[%lu] [JPG] Decoding complete\n", millis());
+  file.close();
+
+  // Write cache file if caching was enabled
+  if (caching) {
+    cache.writeToFile(config.cachePath);
+  }
+
+  return true;
+}
+
+unsigned char JpegToFramebufferConverter::jpegReadCallback(unsigned char* pBuf, unsigned char buf_size,
+                                                           unsigned char* pBytes_actually_read, void* pCallback_data) {
+  JpegContext* context = reinterpret_cast<JpegContext*>(pCallback_data);
+
+  if (context->bufferPos >= context->bufferFilled) {
+    int readCount = context->file.read(context->buffer, sizeof(context->buffer));
+    if (readCount <= 0) {
+      *pBytes_actually_read = 0;
+      return 0;
+    }
+    context->bufferFilled = readCount;
+    context->bufferPos = 0;
+  }
+
+  unsigned int bytesAvailable = context->bufferFilled - context->bufferPos;
+  unsigned int bytesToCopy = (bytesAvailable < buf_size) ? bytesAvailable : buf_size;
+
+  memcpy(pBuf, &context->buffer[context->bufferPos], bytesToCopy);
+  context->bufferPos += bytesToCopy;
+  *pBytes_actually_read = bytesToCopy;
+
+  return 0;
+}
+
+bool JpegToFramebufferConverter::supportsFormat(const std::string& extension) {
+  std::string ext = extension;
+  for (auto& c : ext) {
+    c = tolower(c);
+  }
+  return (ext == ".jpg" || ext == ".jpeg");
+}