feat: add png jpeg support (#556)

## Summary
- Add embedded image support to EPUB rendering with JPEG and PNG
decoders
- Implement pixel caching system to cache decoded/dithered images to SD
card for faster re-rendering
- Add 4-level grayscale support for display
## Changes
### New Image Rendering System
- Add `ImageBlock` class to represent an image with its cached path and
display dimensions
- Add `PageImage` class as a new `PageElement` type for images on pages
- Add `ImageToFramebufferDecoder` interface for format-specific image
decoders
- Add `JpegToFramebufferConverter` - JPEG decoder with Bayer dithering
and scaling
- Add `PngToFramebufferConverter` - PNG decoder with Bayer dithering and
scaling
- Add `ImageDecoderFactory` to select appropriate decoder based on file
extension
- Add `getRenderMode()` to GfxRenderer for grayscale render mode queries
### Dithering and Grayscale
- Implement 4x4 Bayer ordered dithering for 4-level grayscale output
- Stateless algorithm works correctly with MCU block decoding
- Handles scaling without artifacts
- Add grayscale render mode support (BW, GRAYSCALE_LSB, GRAYSCALE_MSB)
- Image decoders and cache renderer respect current render mode
- Enables proper 4-level e-ink grayscale when anti-aliasing is enabled
### Pixel Caching
- Cache decoded/dithered images to `.pxc` files on SD card
- Cache format: 2-bit packed pixels (4 pixels per byte) with
width/height header
- On subsequent renders, load directly from cache instead of re-decoding
- Cache renderer supports grayscale render modes for multi-pass
rendering
- Significantly improves page navigation speed for image-heavy EPUBs
### HTML Parser Integration
- Update `ChapterHtmlSlimParser` to process `<img>` tags and extract
images from EPUB
- Resolve relative image paths within EPUB ZIP structure
- Extract images to cache directory before decoding
- Create `PageImage` elements with proper scaling to fit viewport
- Fall back to alt text display if image processing fails
### Build Configuration
- Add `PNG_MAX_BUFFERED_PIXELS=6402` to support up to 800px wide images

  ### Test Script
                                
  - Generate test EPUBs with annotated JPEG and PNG images
- Test cases cover: grayscale (4 levels), centering, scaling, cache
performance
  
## Test plan
- [x] Open EPUB with JPEG images - verify images display with proper
grayscale
- [x] Open EPUB with PNG images - verify images display correctly and no
crash
- [x] Navigate away from image page and back - verify faster load from
cache
- [x] Verify grayscale tones render correctly (not just black/white
dithering)
- [x] Verify large images are scaled down to fit screen
- [x] Verify images are centered horizontally
- [x] Verify page serialization/deserialization works with images
  - [x] Verify images rendered in landscape mode        

## Test Results
[png](https://photos.app.goo.gl/5zFUb8xA8db3dPd19)
[jpeg](https://photos.app.goo.gl/SwtwaL2DSQwKybhw7)


![20260128_231123790](https://github.com/user-attachments/assets/78855971-4bb8-441a-b207-0a292b9739f5)

![20260128_231012253](https://github.com/user-attachments/assets/f08fb63f-1b73-41d9-a25e-78232ec0c495)

![20260128_231004209](https://github.com/user-attachments/assets/06c94acc-8a06-4955-978e-6e583399478d)

![20260128_230954997](https://github.com/user-attachments/assets/49bc44d5-0f2c-416b-9199-4d680fb0f4c3)

![20260128_230945717](https://github.com/user-attachments/assets/93446da5-2e07-410c-89c9-6a21d14e5acb)

![20260128_230938313](https://github.com/user-attachments/assets/4c74c72a-3d40-4a25-b0f3-acc703f42c00)

![20260128_230925546](https://github.com/user-attachments/assets/8d8f62ee-c8fc-4f19-a12c-da29083bb766)

![20260128_230918374](https://github.com/user-attachments/assets/f007d5db-41cc-4fa6-bb22-9e767ee7b00d)


                                                                       
---

### AI Usage

Did you use AI tools to help write this code? _**< YES  >**_

---------

Co-authored-by: Matthías Páll Gissurarson <mpg@mpg.is>
Co-authored-by: Dave Allie <dave@daveallie.com>
This commit is contained in:
martin brook
2026-02-16 19:56:59 +11:00
committed by GitHub
co-authored by Matthías Páll Gissurarson Dave Allie
parent 5ec4210f23
commit ddc31546db
26 changed files with 2060 additions and 32 deletions
+40
View File
@@ -0,0 +1,40 @@
#pragma once
#include <GfxRenderer.h>
#include <stdint.h>
// 4x4 Bayer matrix for ordered dithering
inline const uint8_t bayer4x4[4][4] = {
{0, 8, 2, 10},
{12, 4, 14, 6},
{3, 11, 1, 9},
{15, 7, 13, 5},
};
// Apply Bayer dithering and quantize to 4 levels (0-3)
// Stateless - works correctly with any pixel processing order
inline uint8_t applyBayerDither4Level(uint8_t gray, int x, int y) {
int bayer = bayer4x4[y & 3][x & 3];
int dither = (bayer - 8) * 5; // Scale to +/-40 (half of quantization step 85)
int adjusted = gray + dither;
if (adjusted < 0) adjusted = 0;
if (adjusted > 255) adjusted = 255;
if (adjusted < 64) return 0;
if (adjusted < 128) return 1;
if (adjusted < 192) return 2;
return 3;
}
// Draw a pixel respecting the current render mode for grayscale support
inline void drawPixelWithRenderMode(GfxRenderer& renderer, int x, int y, uint8_t pixelValue) {
GfxRenderer::RenderMode renderMode = renderer.getRenderMode();
if (renderMode == GfxRenderer::BW && pixelValue < 3) {
renderer.drawPixel(x, y, true);
} else if (renderMode == GfxRenderer::GRAYSCALE_MSB && (pixelValue == 1 || pixelValue == 2)) {
renderer.drawPixel(x, y, false);
} else if (renderMode == GfxRenderer::GRAYSCALE_LSB && pixelValue == 1) {
renderer.drawPixel(x, y, false);
}
}
@@ -0,0 +1,42 @@
#include "ImageDecoderFactory.h"
#include <HardwareSerial.h>
#include <memory>
#include <string>
#include "JpegToFramebufferConverter.h"
#include "PngToFramebufferConverter.h"
std::unique_ptr<JpegToFramebufferConverter> ImageDecoderFactory::jpegDecoder = nullptr;
std::unique_ptr<PngToFramebufferConverter> ImageDecoderFactory::pngDecoder = nullptr;
ImageToFramebufferDecoder* ImageDecoderFactory::getDecoder(const std::string& imagePath) {
std::string ext = imagePath;
size_t dotPos = ext.rfind('.');
if (dotPos != std::string::npos) {
ext = ext.substr(dotPos);
for (auto& c : ext) {
c = tolower(c);
}
} else {
ext = "";
}
if (JpegToFramebufferConverter::supportsFormat(ext)) {
if (!jpegDecoder) {
jpegDecoder.reset(new JpegToFramebufferConverter());
}
return jpegDecoder.get();
} else if (PngToFramebufferConverter::supportsFormat(ext)) {
if (!pngDecoder) {
pngDecoder.reset(new PngToFramebufferConverter());
}
return pngDecoder.get();
}
Serial.printf("[%lu] [DEC] No decoder found for image: %s\n", millis(), imagePath.c_str());
return nullptr;
}
bool ImageDecoderFactory::isFormatSupported(const std::string& imagePath) { return getDecoder(imagePath) != nullptr; }
@@ -0,0 +1,20 @@
#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include "ImageToFramebufferDecoder.h"
class JpegToFramebufferConverter;
class PngToFramebufferConverter;
class ImageDecoderFactory {
public:
// Returns non-owning pointer - factory owns the decoder lifetime
static ImageToFramebufferDecoder* getDecoder(const std::string& imagePath);
static bool isFormatSupported(const std::string& imagePath);
private:
static std::unique_ptr<JpegToFramebufferConverter> jpegDecoder;
static std::unique_ptr<PngToFramebufferConverter> pngDecoder;
};
@@ -0,0 +1,18 @@
#include "ImageToFramebufferDecoder.h"
#include <Arduino.h>
#include <HardwareSerial.h>
bool ImageToFramebufferDecoder::validateImageDimensions(int width, int height, const std::string& format) {
if (width * height > MAX_SOURCE_PIXELS) {
Serial.printf("[%lu] [IMG] Image too large (%dx%d = %d pixels %s), max supported: %d pixels\n", millis(), width,
height, width * height, format.c_str(), MAX_SOURCE_PIXELS);
return false;
}
return true;
}
void ImageToFramebufferDecoder::warnUnsupportedFeature(const std::string& feature, const std::string& imagePath) {
Serial.printf("[%lu] [IMG] Warning: Unsupported feature '%s' in image '%s'. Image may not display correctly.\n",
millis(), feature.c_str(), imagePath.c_str());
}
@@ -0,0 +1,40 @@
#pragma once
#include <SdFat.h>
#include <memory>
#include <string>
class GfxRenderer;
struct ImageDimensions {
int16_t width;
int16_t height;
};
struct RenderConfig {
int x, y;
int maxWidth, maxHeight;
bool useGrayscale = true;
bool useDithering = true;
bool performanceMode = false;
bool useExactDimensions = false; // If true, use maxWidth/maxHeight as exact output size (no recalculation)
std::string cachePath; // If non-empty, decoder will write pixel cache to this path
};
class ImageToFramebufferDecoder {
public:
virtual ~ImageToFramebufferDecoder() = default;
virtual bool decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer, const RenderConfig& config) = 0;
virtual bool getDimensions(const std::string& imagePath, ImageDimensions& dims) const = 0;
virtual const char* getFormatName() const = 0;
protected:
// Size validation helpers
static constexpr int MAX_SOURCE_PIXELS = 3145728; // 2048 * 1536
bool validateImageDimensions(int width, int height, const std::string& format);
void warnUnsupportedFeature(const std::string& feature, const std::string& imagePath);
};
@@ -0,0 +1,298 @@
#include "JpegToFramebufferConverter.h"
#include <GfxRenderer.h>
#include <HardwareSerial.h>
#include <SDCardManager.h>
#include <SdFat.h>
#include <picojpeg.h>
#include <cstdio>
#include <cstring>
#include "DitherUtils.h"
#include "PixelCache.h"
struct JpegContext {
FsFile& file;
uint8_t buffer[512];
size_t bufferPos;
size_t bufferFilled;
JpegContext(FsFile& f) : file(f), bufferPos(0), bufferFilled(0) {}
};
bool JpegToFramebufferConverter::getDimensionsStatic(const std::string& imagePath, ImageDimensions& out) {
FsFile file;
if (!Storage.openFileForRead("JPG", imagePath, file)) {
Serial.printf("[%lu] [JPG] Failed to open file for dimensions: %s\n", millis(), imagePath.c_str());
return false;
}
JpegContext context(file);
pjpeg_image_info_t imageInfo;
int status = pjpeg_decode_init(&imageInfo, jpegReadCallback, &context, 0);
file.close();
if (status != 0) {
Serial.printf("[%lu] [JPG] Failed to init JPEG for dimensions: %d\n", millis(), status);
return false;
}
out.width = imageInfo.m_width;
out.height = imageInfo.m_height;
Serial.printf("[%lu] [JPG] Image dimensions: %dx%d\n", millis(), out.width, out.height);
return true;
}
bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer,
const RenderConfig& config) {
Serial.printf("[%lu] [JPG] Decoding JPEG: %s\n", millis(), imagePath.c_str());
FsFile file;
if (!Storage.openFileForRead("JPG", imagePath, file)) {
Serial.printf("[%lu] [JPG] Failed to open file: %s\n", millis(), imagePath.c_str());
return false;
}
JpegContext context(file);
pjpeg_image_info_t imageInfo;
int status = pjpeg_decode_init(&imageInfo, jpegReadCallback, &context, 0);
if (status != 0) {
Serial.printf("[%lu] [JPG] picojpeg init failed: %d\n", millis(), status);
file.close();
return false;
}
if (!validateImageDimensions(imageInfo.m_width, imageInfo.m_height, "JPEG")) {
file.close();
return false;
}
// Calculate output dimensions
int destWidth, destHeight;
float scale;
if (config.useExactDimensions && config.maxWidth > 0 && config.maxHeight > 0) {
// Use exact dimensions as specified (avoids rounding mismatches with pre-calculated sizes)
destWidth = config.maxWidth;
destHeight = config.maxHeight;
scale = (float)destWidth / imageInfo.m_width;
} else {
// Calculate scale factor to fit within maxWidth/maxHeight
float scaleX = (config.maxWidth > 0 && imageInfo.m_width > config.maxWidth)
? (float)config.maxWidth / imageInfo.m_width
: 1.0f;
float scaleY = (config.maxHeight > 0 && imageInfo.m_height > config.maxHeight)
? (float)config.maxHeight / imageInfo.m_height
: 1.0f;
scale = (scaleX < scaleY) ? scaleX : scaleY;
if (scale > 1.0f) scale = 1.0f;
destWidth = (int)(imageInfo.m_width * scale);
destHeight = (int)(imageInfo.m_height * scale);
}
Serial.printf("[%lu] [JPG] JPEG %dx%d -> %dx%d (scale %.2f), scan type: %d, MCU: %dx%d\n", millis(),
imageInfo.m_width, imageInfo.m_height, destWidth, destHeight, scale, imageInfo.m_scanType,
imageInfo.m_MCUWidth, imageInfo.m_MCUHeight);
if (!imageInfo.m_pMCUBufR || !imageInfo.m_pMCUBufG || !imageInfo.m_pMCUBufB) {
Serial.printf("[%lu] [JPG] Null buffer pointers in imageInfo\n", millis());
file.close();
return false;
}
const int screenWidth = renderer.getScreenWidth();
const int screenHeight = renderer.getScreenHeight();
// Allocate pixel cache if cachePath is provided
PixelCache cache;
bool caching = !config.cachePath.empty();
if (caching) {
if (!cache.allocate(destWidth, destHeight, config.x, config.y)) {
Serial.printf("[%lu] [JPG] Failed to allocate cache buffer, continuing without caching\n", millis());
caching = false;
}
}
int mcuX = 0;
int mcuY = 0;
while (mcuY < imageInfo.m_MCUSPerCol) {
status = pjpeg_decode_mcu();
if (status == PJPG_NO_MORE_BLOCKS) {
break;
}
if (status != 0) {
Serial.printf("[%lu] [JPG] MCU decode failed: %d\n", millis(), status);
file.close();
return false;
}
// Source position in image coordinates
int srcStartX = mcuX * imageInfo.m_MCUWidth;
int srcStartY = mcuY * imageInfo.m_MCUHeight;
switch (imageInfo.m_scanType) {
case PJPG_GRAYSCALE:
for (int row = 0; row < 8; row++) {
int srcY = srcStartY + row;
int destY = config.y + (int)(srcY * scale);
if (destY >= screenHeight || destY >= config.y + destHeight) continue;
for (int col = 0; col < 8; col++) {
int srcX = srcStartX + col;
int destX = config.x + (int)(srcX * scale);
if (destX >= screenWidth || destX >= config.x + destWidth) continue;
uint8_t gray = imageInfo.m_pMCUBufR[row * 8 + col];
uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
if (dithered > 3) dithered = 3;
drawPixelWithRenderMode(renderer, destX, destY, dithered);
if (caching) cache.setPixel(destX, destY, dithered);
}
}
break;
case PJPG_YH1V1:
for (int row = 0; row < 8; row++) {
int srcY = srcStartY + row;
int destY = config.y + (int)(srcY * scale);
if (destY >= screenHeight || destY >= config.y + destHeight) continue;
for (int col = 0; col < 8; col++) {
int srcX = srcStartX + col;
int destX = config.x + (int)(srcX * scale);
if (destX >= screenWidth || destX >= config.x + destWidth) continue;
uint8_t r = imageInfo.m_pMCUBufR[row * 8 + col];
uint8_t g = imageInfo.m_pMCUBufG[row * 8 + col];
uint8_t b = imageInfo.m_pMCUBufB[row * 8 + col];
uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
if (dithered > 3) dithered = 3;
drawPixelWithRenderMode(renderer, destX, destY, dithered);
if (caching) cache.setPixel(destX, destY, dithered);
}
}
break;
case PJPG_YH2V1:
for (int row = 0; row < 8; row++) {
int srcY = srcStartY + row;
int destY = config.y + (int)(srcY * scale);
if (destY >= screenHeight || destY >= config.y + destHeight) continue;
for (int col = 0; col < 16; col++) {
int srcX = srcStartX + col;
int destX = config.x + (int)(srcX * scale);
if (destX >= screenWidth || destX >= config.x + destWidth) continue;
int blockIndex = (col < 8) ? 0 : 1;
int pixelIndex = row * 8 + (col % 8);
uint8_t r = imageInfo.m_pMCUBufR[blockIndex * 64 + pixelIndex];
uint8_t g = imageInfo.m_pMCUBufG[blockIndex * 64 + pixelIndex];
uint8_t b = imageInfo.m_pMCUBufB[blockIndex * 64 + pixelIndex];
uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
if (dithered > 3) dithered = 3;
drawPixelWithRenderMode(renderer, destX, destY, dithered);
if (caching) cache.setPixel(destX, destY, dithered);
}
}
break;
case PJPG_YH1V2:
for (int row = 0; row < 16; row++) {
int srcY = srcStartY + row;
int destY = config.y + (int)(srcY * scale);
if (destY >= screenHeight || destY >= config.y + destHeight) continue;
for (int col = 0; col < 8; col++) {
int srcX = srcStartX + col;
int destX = config.x + (int)(srcX * scale);
if (destX >= screenWidth || destX >= config.x + destWidth) continue;
int blockIndex = (row < 8) ? 0 : 1;
int pixelIndex = (row % 8) * 8 + col;
uint8_t r = imageInfo.m_pMCUBufR[blockIndex * 128 + pixelIndex];
uint8_t g = imageInfo.m_pMCUBufG[blockIndex * 128 + pixelIndex];
uint8_t b = imageInfo.m_pMCUBufB[blockIndex * 128 + pixelIndex];
uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
if (dithered > 3) dithered = 3;
drawPixelWithRenderMode(renderer, destX, destY, dithered);
if (caching) cache.setPixel(destX, destY, dithered);
}
}
break;
case PJPG_YH2V2:
for (int row = 0; row < 16; row++) {
int srcY = srcStartY + row;
int destY = config.y + (int)(srcY * scale);
if (destY >= screenHeight || destY >= config.y + destHeight) continue;
for (int col = 0; col < 16; col++) {
int srcX = srcStartX + col;
int destX = config.x + (int)(srcX * scale);
if (destX >= screenWidth || destX >= config.x + destWidth) continue;
int blockX = (col < 8) ? 0 : 1;
int blockY = (row < 8) ? 0 : 1;
int blockIndex = blockY * 2 + blockX;
int pixelIndex = (row % 8) * 8 + (col % 8);
int blockOffset = blockIndex * 64;
uint8_t r = imageInfo.m_pMCUBufR[blockOffset + pixelIndex];
uint8_t g = imageInfo.m_pMCUBufG[blockOffset + pixelIndex];
uint8_t b = imageInfo.m_pMCUBufB[blockOffset + pixelIndex];
uint8_t gray = (uint8_t)((r * 77 + g * 150 + b * 29) >> 8);
uint8_t dithered = config.useDithering ? applyBayerDither4Level(gray, destX, destY) : gray / 85;
if (dithered > 3) dithered = 3;
drawPixelWithRenderMode(renderer, destX, destY, dithered);
if (caching) cache.setPixel(destX, destY, dithered);
}
}
break;
}
mcuX++;
if (mcuX >= imageInfo.m_MCUSPerRow) {
mcuX = 0;
mcuY++;
}
}
Serial.printf("[%lu] [JPG] Decoding complete\n", millis());
file.close();
// Write cache file if caching was enabled
if (caching) {
cache.writeToFile(config.cachePath);
}
return true;
}
unsigned char JpegToFramebufferConverter::jpegReadCallback(unsigned char* pBuf, unsigned char buf_size,
unsigned char* pBytes_actually_read, void* pCallback_data) {
JpegContext* context = reinterpret_cast<JpegContext*>(pCallback_data);
if (context->bufferPos >= context->bufferFilled) {
int readCount = context->file.read(context->buffer, sizeof(context->buffer));
if (readCount <= 0) {
*pBytes_actually_read = 0;
return 0;
}
context->bufferFilled = readCount;
context->bufferPos = 0;
}
unsigned int bytesAvailable = context->bufferFilled - context->bufferPos;
unsigned int bytesToCopy = (bytesAvailable < buf_size) ? bytesAvailable : buf_size;
memcpy(pBuf, &context->buffer[context->bufferPos], bytesToCopy);
context->bufferPos += bytesToCopy;
*pBytes_actually_read = bytesToCopy;
return 0;
}
bool JpegToFramebufferConverter::supportsFormat(const std::string& extension) {
std::string ext = extension;
for (auto& c : ext) {
c = tolower(c);
}
return (ext == ".jpg" || ext == ".jpeg");
}
@@ -0,0 +1,24 @@
#pragma once
#include <stdint.h>
#include <string>
#include "ImageToFramebufferDecoder.h"
class JpegToFramebufferConverter final : public ImageToFramebufferDecoder {
public:
static bool getDimensionsStatic(const std::string& imagePath, ImageDimensions& out);
bool decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer, const RenderConfig& config) override;
bool getDimensions(const std::string& imagePath, ImageDimensions& dims) const override {
return getDimensionsStatic(imagePath, dims);
}
static bool supportsFormat(const std::string& extension);
const char* getFormatName() const override { return "JPEG"; }
private:
static unsigned char jpegReadCallback(unsigned char* pBuf, unsigned char buf_size,
unsigned char* pBytes_actually_read, void* pCallback_data);
};
+85
View File
@@ -0,0 +1,85 @@
#pragma once
#include <HardwareSerial.h>
#include <SDCardManager.h>
#include <SdFat.h>
#include <stdint.h>
#include <cstring>
#include <string>
// Cache buffer for storing 2-bit pixels (4 levels) during decode.
// Packs 4 pixels per byte, MSB first.
struct PixelCache {
uint8_t* buffer;
int width;
int height;
int bytesPerRow;
int originX; // config.x - to convert screen coords to cache coords
int originY; // config.y
PixelCache() : buffer(nullptr), width(0), height(0), bytesPerRow(0), originX(0), originY(0) {}
PixelCache(const PixelCache&) = delete;
PixelCache& operator=(const PixelCache&) = delete;
static constexpr size_t MAX_CACHE_BYTES = 256 * 1024; // 256KB limit for embedded targets
bool allocate(int w, int h, int ox, int oy) {
width = w;
height = h;
originX = ox;
originY = oy;
bytesPerRow = (w + 3) / 4; // 2 bits per pixel, 4 pixels per byte
size_t bufferSize = (size_t)bytesPerRow * h;
if (bufferSize > MAX_CACHE_BYTES) {
Serial.printf("[%lu] [IMG] Cache buffer too large: %d bytes for %dx%d (limit %d)\n", millis(), bufferSize, w, h,
MAX_CACHE_BYTES);
return false;
}
buffer = (uint8_t*)malloc(bufferSize);
if (buffer) {
memset(buffer, 0, bufferSize);
Serial.printf("[%lu] [IMG] Allocated cache buffer: %d bytes for %dx%d\n", millis(), bufferSize, w, h);
}
return buffer != nullptr;
}
void setPixel(int screenX, int screenY, uint8_t value) {
if (!buffer) return;
int localX = screenX - originX;
int localY = screenY - originY;
if (localX < 0 || localX >= width || localY < 0 || localY >= height) return;
int byteIdx = localY * bytesPerRow + localX / 4;
int bitShift = 6 - (localX % 4) * 2; // MSB first: pixel 0 at bits 6-7
buffer[byteIdx] = (buffer[byteIdx] & ~(0x03 << bitShift)) | ((value & 0x03) << bitShift);
}
bool writeToFile(const std::string& cachePath) {
if (!buffer) return false;
FsFile cacheFile;
if (!Storage.openFileForWrite("IMG", cachePath, cacheFile)) {
Serial.printf("[%lu] [IMG] Failed to open cache file for writing: %s\n", millis(), cachePath.c_str());
return false;
}
uint16_t w = width;
uint16_t h = height;
cacheFile.write(&w, 2);
cacheFile.write(&h, 2);
cacheFile.write(buffer, bytesPerRow * height);
cacheFile.close();
Serial.printf("[%lu] [IMG] Cache written: %s (%dx%d, %d bytes)\n", millis(), cachePath.c_str(), width, height,
4 + bytesPerRow * height);
return true;
}
~PixelCache() {
if (buffer) {
free(buffer);
buffer = nullptr;
}
}
};
@@ -0,0 +1,364 @@
#include "PngToFramebufferConverter.h"
#include <GfxRenderer.h>
#include <HardwareSerial.h>
#include <PNGdec.h>
#include <SDCardManager.h>
#include <SdFat.h>
#include <cstdlib>
#include <new>
#include "DitherUtils.h"
#include "PixelCache.h"
namespace {
// Context struct passed through PNGdec callbacks to avoid global mutable state.
// The draw callback receives this via pDraw->pUser (set by png.decode()).
// The file I/O callbacks receive the FsFile* via pFile->fHandle (set by pngOpen()).
struct PngContext {
GfxRenderer* renderer;
const RenderConfig* config;
int screenWidth;
int screenHeight;
// Scaling state
float scale;
int srcWidth;
int srcHeight;
int dstWidth;
int dstHeight;
int lastDstY; // Track last rendered destination Y to avoid duplicates
PixelCache cache;
bool caching;
uint8_t* grayLineBuffer;
PngContext()
: renderer(nullptr),
config(nullptr),
screenWidth(0),
screenHeight(0),
scale(1.0f),
srcWidth(0),
srcHeight(0),
dstWidth(0),
dstHeight(0),
lastDstY(-1),
caching(false),
grayLineBuffer(nullptr) {}
};
// File I/O callbacks use pFile->fHandle to access the FsFile*,
// avoiding the need for global file state.
void* pngOpenWithHandle(const char* filename, int32_t* size) {
FsFile* f = new FsFile();
if (!Storage.openFileForRead("PNG", std::string(filename), *f)) {
delete f;
return nullptr;
}
*size = f->size();
return f;
}
void pngCloseWithHandle(void* handle) {
FsFile* f = reinterpret_cast<FsFile*>(handle);
if (f) {
f->close();
delete f;
}
}
int32_t pngReadWithHandle(PNGFILE* pFile, uint8_t* pBuf, int32_t len) {
FsFile* f = reinterpret_cast<FsFile*>(pFile->fHandle);
if (!f) return 0;
return f->read(pBuf, len);
}
int32_t pngSeekWithHandle(PNGFILE* pFile, int32_t pos) {
FsFile* f = reinterpret_cast<FsFile*>(pFile->fHandle);
if (!f) return -1;
return f->seek(pos);
}
// The PNG decoder (PNGdec) is ~42 KB due to internal zlib decompression buffers.
// We heap-allocate it on demand rather than using a static instance, so this memory
// is only consumed while actually decoding/querying PNG images. This is critical on
// the ESP32-C3 where total RAM is ~320 KB.
constexpr size_t PNG_DECODER_APPROX_SIZE = 44 * 1024; // ~42 KB + overhead
constexpr size_t MIN_FREE_HEAP_FOR_PNG = PNG_DECODER_APPROX_SIZE + 16 * 1024; // decoder + 16 KB headroom
// Convert entire source line to grayscale with alpha blending to white background.
// For indexed PNGs with tRNS chunk, alpha values are stored at palette[768] onwards.
// Processing the whole line at once improves cache locality and reduces per-pixel overhead.
void convertLineToGray(uint8_t* pPixels, uint8_t* grayLine, int width, int pixelType, uint8_t* palette, int hasAlpha) {
switch (pixelType) {
case PNG_PIXEL_GRAYSCALE:
memcpy(grayLine, pPixels, width);
break;
case PNG_PIXEL_TRUECOLOR:
for (int x = 0; x < width; x++) {
uint8_t* p = &pPixels[x * 3];
grayLine[x] = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
}
break;
case PNG_PIXEL_INDEXED:
if (palette) {
if (hasAlpha) {
for (int x = 0; x < width; x++) {
uint8_t idx = pPixels[x];
uint8_t* p = &palette[idx * 3];
uint8_t gray = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
uint8_t alpha = palette[768 + idx];
grayLine[x] = (uint8_t)((gray * alpha + 255 * (255 - alpha)) / 255);
}
} else {
for (int x = 0; x < width; x++) {
uint8_t* p = &palette[pPixels[x] * 3];
grayLine[x] = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
}
}
} else {
memcpy(grayLine, pPixels, width);
}
break;
case PNG_PIXEL_GRAY_ALPHA:
for (int x = 0; x < width; x++) {
uint8_t gray = pPixels[x * 2];
uint8_t alpha = pPixels[x * 2 + 1];
grayLine[x] = (uint8_t)((gray * alpha + 255 * (255 - alpha)) / 255);
}
break;
case PNG_PIXEL_TRUECOLOR_ALPHA:
for (int x = 0; x < width; x++) {
uint8_t* p = &pPixels[x * 4];
uint8_t gray = (uint8_t)((p[0] * 77 + p[1] * 150 + p[2] * 29) >> 8);
uint8_t alpha = p[3];
grayLine[x] = (uint8_t)((gray * alpha + 255 * (255 - alpha)) / 255);
}
break;
default:
memset(grayLine, 128, width);
break;
}
}
int pngDrawCallback(PNGDRAW* pDraw) {
PngContext* ctx = reinterpret_cast<PngContext*>(pDraw->pUser);
if (!ctx || !ctx->config || !ctx->renderer || !ctx->grayLineBuffer) return 0;
int srcY = pDraw->y;
int srcWidth = ctx->srcWidth;
// Calculate destination Y with scaling
int dstY = (int)(srcY * ctx->scale);
// Skip if we already rendered this destination row (multiple source rows map to same dest)
if (dstY == ctx->lastDstY) return 1;
ctx->lastDstY = dstY;
// Check bounds
if (dstY >= ctx->dstHeight) return 1;
int outY = ctx->config->y + dstY;
if (outY >= ctx->screenHeight) return 1;
// Convert entire source line to grayscale (improves cache locality)
convertLineToGray(pDraw->pPixels, ctx->grayLineBuffer, srcWidth, pDraw->iPixelType, pDraw->pPalette,
pDraw->iHasAlpha);
// Render scaled row using Bresenham-style integer stepping (no floating-point division)
int dstWidth = ctx->dstWidth;
int outXBase = ctx->config->x;
int screenWidth = ctx->screenWidth;
bool useDithering = ctx->config->useDithering;
bool caching = ctx->caching;
int srcX = 0;
int error = 0;
for (int dstX = 0; dstX < dstWidth; dstX++) {
int outX = outXBase + dstX;
if (outX < screenWidth) {
uint8_t gray = ctx->grayLineBuffer[srcX];
uint8_t ditheredGray;
if (useDithering) {
ditheredGray = applyBayerDither4Level(gray, outX, outY);
} else {
ditheredGray = gray / 85;
if (ditheredGray > 3) ditheredGray = 3;
}
drawPixelWithRenderMode(*ctx->renderer, outX, outY, ditheredGray);
if (caching) ctx->cache.setPixel(outX, outY, ditheredGray);
}
// Bresenham-style stepping: advance srcX based on ratio srcWidth/dstWidth
error += srcWidth;
while (error >= dstWidth) {
error -= dstWidth;
srcX++;
}
}
return 1;
}
} // namespace
bool PngToFramebufferConverter::getDimensionsStatic(const std::string& imagePath, ImageDimensions& out) {
size_t freeHeap = ESP.getFreeHeap();
if (freeHeap < MIN_FREE_HEAP_FOR_PNG) {
Serial.printf("[%lu] [PNG] Not enough heap for PNG decoder (%u free, need %u)\n", millis(), freeHeap,
MIN_FREE_HEAP_FOR_PNG);
return false;
}
PNG* png = new (std::nothrow) PNG();
if (!png) {
Serial.printf("[%lu] [PNG] Failed to allocate PNG decoder for dimensions\n", millis());
return false;
}
int rc = png->open(imagePath.c_str(), pngOpenWithHandle, pngCloseWithHandle, pngReadWithHandle, pngSeekWithHandle,
nullptr);
if (rc != 0) {
Serial.printf("[%lu] [PNG] Failed to open PNG for dimensions: %d\n", millis(), rc);
delete png;
return false;
}
out.width = png->getWidth();
out.height = png->getHeight();
png->close();
delete png;
return true;
}
bool PngToFramebufferConverter::decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer,
const RenderConfig& config) {
Serial.printf("[%lu] [PNG] Decoding PNG: %s\n", millis(), imagePath.c_str());
size_t freeHeap = ESP.getFreeHeap();
if (freeHeap < MIN_FREE_HEAP_FOR_PNG) {
Serial.printf("[%lu] [PNG] Not enough heap for PNG decoder (%u free, need %u)\n", millis(), freeHeap,
MIN_FREE_HEAP_FOR_PNG);
return false;
}
// Heap-allocate PNG decoder (~42 KB) - freed at end of function
PNG* png = new (std::nothrow) PNG();
if (!png) {
Serial.printf("[%lu] [PNG] Failed to allocate PNG decoder\n", millis());
return false;
}
PngContext ctx;
ctx.renderer = &renderer;
ctx.config = &config;
ctx.screenWidth = renderer.getScreenWidth();
ctx.screenHeight = renderer.getScreenHeight();
int rc = png->open(imagePath.c_str(), pngOpenWithHandle, pngCloseWithHandle, pngReadWithHandle, pngSeekWithHandle,
pngDrawCallback);
if (rc != PNG_SUCCESS) {
Serial.printf("[%lu] [PNG] Failed to open PNG: %d\n", millis(), rc);
delete png;
return false;
}
if (!validateImageDimensions(png->getWidth(), png->getHeight(), "PNG")) {
png->close();
delete png;
return false;
}
// Calculate output dimensions
ctx.srcWidth = png->getWidth();
ctx.srcHeight = png->getHeight();
if (config.useExactDimensions && config.maxWidth > 0 && config.maxHeight > 0) {
// Use exact dimensions as specified (avoids rounding mismatches with pre-calculated sizes)
ctx.dstWidth = config.maxWidth;
ctx.dstHeight = config.maxHeight;
ctx.scale = (float)ctx.dstWidth / ctx.srcWidth;
} else {
// Calculate scale factor to fit within maxWidth/maxHeight
float scaleX = (float)config.maxWidth / ctx.srcWidth;
float scaleY = (float)config.maxHeight / ctx.srcHeight;
ctx.scale = (scaleX < scaleY) ? scaleX : scaleY;
if (ctx.scale > 1.0f) ctx.scale = 1.0f; // Don't upscale
ctx.dstWidth = (int)(ctx.srcWidth * ctx.scale);
ctx.dstHeight = (int)(ctx.srcHeight * ctx.scale);
}
ctx.lastDstY = -1; // Reset row tracking
Serial.printf("[%lu] [PNG] PNG %dx%d -> %dx%d (scale %.2f), bpp: %d\n", millis(), ctx.srcWidth, ctx.srcHeight,
ctx.dstWidth, ctx.dstHeight, ctx.scale, png->getBpp());
if (png->getBpp() != 8) {
warnUnsupportedFeature("bit depth (" + std::to_string(png->getBpp()) + "bpp)", imagePath);
}
// Allocate grayscale line buffer on demand (~3.2 KB) - freed after decode
const size_t grayBufSize = PNG_MAX_BUFFERED_PIXELS / 2;
ctx.grayLineBuffer = static_cast<uint8_t*>(malloc(grayBufSize));
if (!ctx.grayLineBuffer) {
Serial.printf("[%lu] [PNG] Failed to allocate gray line buffer\n", millis());
png->close();
delete png;
return false;
}
// Allocate cache buffer using SCALED dimensions
ctx.caching = !config.cachePath.empty();
if (ctx.caching) {
if (!ctx.cache.allocate(ctx.dstWidth, ctx.dstHeight, config.x, config.y)) {
Serial.printf("[%lu] [PNG] Failed to allocate cache buffer, continuing without caching\n", millis());
ctx.caching = false;
}
}
unsigned long decodeStart = millis();
rc = png->decode(&ctx, 0);
unsigned long decodeTime = millis() - decodeStart;
free(ctx.grayLineBuffer);
ctx.grayLineBuffer = nullptr;
if (rc != PNG_SUCCESS) {
Serial.printf("[%lu] [PNG] Decode failed: %d\n", millis(), rc);
png->close();
delete png;
return false;
}
png->close();
delete png;
Serial.printf("[%lu] [PNG] PNG decoding complete - render time: %lu ms\n", millis(), decodeTime);
// Write cache file if caching was enabled and buffer was allocated
if (ctx.caching) {
ctx.cache.writeToFile(config.cachePath);
}
return true;
}
bool PngToFramebufferConverter::supportsFormat(const std::string& extension) {
std::string ext = extension;
for (auto& c : ext) {
c = tolower(c);
}
return (ext == ".png");
}
@@ -0,0 +1,17 @@
#pragma once
#include "ImageToFramebufferDecoder.h"
class PngToFramebufferConverter final : public ImageToFramebufferDecoder {
public:
static bool getDimensionsStatic(const std::string& imagePath, ImageDimensions& out);
bool decodeToFramebuffer(const std::string& imagePath, GfxRenderer& renderer, const RenderConfig& config) override;
bool getDimensions(const std::string& imagePath, ImageDimensions& dims) const override {
return getDimensionsStatic(imagePath, dims);
}
static bool supportsFormat(const std::string& extension);
const char* getFormatName() const override { return "PNG"; }
};