feat: add SD card font support with on-device download and web management

Add a complete SD card font subsystem that enables users to install and
use custom fonts beyond the three built-in families. This combines the
back-end firmware support (#1327) with the font configuration, build
pipeline, CI distribution, and user-facing management UI (#1392).

Core font system:
- Custom .cpfont binary format (v4) with multi-style support (regular,
  bold, italic, bold-italic) packed into a single file per size
- On-demand glyph loading from SD card with two-pass prewarm rendering
  to bulk-read glyphs per page, achieving near-flash performance for
  Latin text (~697ms vs ~681ms) and viable CJK rendering (~32% slower)
- Persistent advance cache for layout measurement without SD I/O
- Overflow ring buffer for glyph cache misses during rendering
- Memory-conscious design: only advance tables kept in RAM; glyph
  bitmaps, kern tables, and ligatures loaded on demand from SD

Font management:
- On-device WiFi download from GitHub Releases with manifest-based
  discovery, install/update detection, and progress UI
- Web interface font upload, listing, and deletion via /fonts page
- Manual SD card copy to /fonts/ or /.fonts/ directories
- Font selection integrated into Settings > Reader > Font Family

Build pipeline:
- Declarative YAML config (sd-fonts.yaml) as single source of truth
  for the 17-family font library (serif, sans, mono, accessibility)
- Python converter (fontconvert_sdcard.py) for TTF/OTF to .cpfont with
  FreeType rasterization, class-based kerning, and ligature extraction
- Parallel build orchestrator with variable font instance extraction
- CI workflow publishing versioned + stable releases to a dedicated
  crosspoint-fonts repository with auto-incrementing revision tags
- Centralized version constants (cpfont_version.py) shared across
  build tooling and CI, with firmware headers as manual sync points

Additional fixes:
- CJK characters no longer get hyphens inserted at line breaks
- Advance table eliminates 30+ second stalls during CJK section
  indexing for paragraphs with >512 unique codepoints

Closes #930

Co-authored-by: Zach Nelson <zach@zdnelson.com>
Co-authored-by: Justin <itsthisjustin@users.noreply.github.com>
Co-authored-by: jpirnay <jens@pirnay.com>
Co-authored-by: mcrosson <kemonine@kemonine.info>
This commit is contained in:
Adrian Wilkins-Caruana
2026-05-08 21:50:06 -05:00
committed by Zach Nelson
co-authored by Zach Nelson Justin jpirnay mcrosson
parent 29fd29f537
commit 7993b2bb97
57 changed files with 6064 additions and 54 deletions
+222 -5
View File
@@ -11,11 +11,15 @@
#include <algorithm>
#include "CrossPointSettings.h"
#include "FontInstaller.h"
#include "OpdsServerStore.h"
#include "SdCardFontGlobals.h"
#include "SdCardFontSystem.h"
#include "SettingsList.h"
#include "WebDAVHandler.h"
#include "WifiCredentialStore.h"
#include "html/FilesPageHtml.generated.h"
#include "html/FontsPageHtml.generated.h"
#include "html/HomePageHtml.generated.h"
#include "html/SettingsPageHtml.generated.h"
#include "html/js/jszip_minJs.generated.h"
@@ -164,6 +168,12 @@ void CrossPointWebServer::begin() {
server->on("/api/settings", HTTP_GET, [this] { handleGetSettings(); });
server->on("/api/settings", HTTP_POST, [this] { handlePostSettings(); });
// Font management endpoints
server->on("/fonts", HTTP_GET, [this] { handleFontsPage(); });
server->on("/api/fonts", HTTP_GET, [this] { handleFontList(); });
server->on("/api/fonts/upload", HTTP_POST, [this] { handleFontUpload(); }, [this] { handleFontUploadData(); });
server->on("/api/fonts/delete", HTTP_POST, [this] { handleFontDelete(); });
// OPDS server endpoints
server->on("/api/opds", HTTP_GET, [this] { handleGetOpdsServers(); });
server->on("/api/opds", HTTP_POST, [this] { handlePostOpdsServer(); });
@@ -1101,7 +1111,10 @@ void CrossPointWebServer::handleSettingsPage() const {
}
void CrossPointWebServer::handleGetSettings() const {
const auto& settings = getSettingsList();
// Pass the SD font registry so the fontFamily setting's enumStringValues
// includes SD-resident families — otherwise the web API only exposes the
// three built-in fonts.
const auto& settings = getSettingsList(&sdFontSystem.registry());
server->setContentLength(CONTENT_LENGTH_UNKNOWN);
server->send(200, "application/json", "");
@@ -1136,8 +1149,14 @@ void CrossPointWebServer::handleGetSettings() const {
doc["value"] = static_cast<int>(s.valueGetter());
}
JsonArray options = doc["options"].to<JsonArray>();
for (const auto& opt : s.enumValues) {
options.add(I18N.get(opt));
if (!s.enumStringValues.empty()) {
for (const auto& opt : s.enumStringValues) {
options.add(opt);
}
} else {
for (const auto& opt : s.enumValues) {
options.add(I18N.get(opt));
}
}
break;
}
@@ -1197,7 +1216,7 @@ void CrossPointWebServer::handlePostSettings() {
return;
}
const auto& settings = getSettingsList();
const auto& settings = getSettingsList(&sdFontSystem.registry());
int applied = 0;
for (const auto& s : settings) {
@@ -1215,7 +1234,9 @@ void CrossPointWebServer::handlePostSettings() {
}
case SettingType::ENUM: {
const int val = doc[s.key].as<int>();
if (val >= 0 && val < static_cast<int>(s.enumValues.size())) {
const int maxVal = s.enumStringValues.empty() ? static_cast<int>(s.enumValues.size())
: static_cast<int>(s.enumStringValues.size());
if (val >= 0 && val < maxVal) {
if (s.valuePtr) {
SETTINGS.*(s.valuePtr) = static_cast<uint8_t>(val);
} else if (s.valueSetter) {
@@ -1694,3 +1715,199 @@ void CrossPointWebServer::onWebSocketEvent(uint8_t num, WStype_t type, uint8_t*
break;
}
}
// --- Font management handlers ---
void CrossPointWebServer::handleFontsPage() const {
sendHtmlContent(server.get(), FontsPageHtml, sizeof(FontsPageHtml));
LOG_DBG("WEB", "Served fonts page");
}
void CrossPointWebServer::handleFontList() const {
// Pick up any uploads/deletes that happened since the last reader load.
const_cast<SdCardFontSystem&>(sdFontSystem).refreshIfDirty();
const auto& families = sdFontSystem.registry().getFamilies();
JsonDocument doc;
JsonArray arr = doc["families"].to<JsonArray>();
doc["maxFamilies"] = SdCardFontRegistry::MAX_SD_FAMILIES;
for (const auto& family : families) {
JsonObject fObj = arr.add<JsonObject>();
fObj["name"] = family.name;
JsonArray sizes = fObj["sizes"].to<JsonArray>();
for (uint8_t s : family.availableSizes()) {
sizes.add(s);
}
JsonArray files = fObj["files"].to<JsonArray>();
for (const auto& file : family.files) {
JsonObject fileObj = files.add<JsonObject>();
// Extract filename from full path
const char* name = strrchr(file.path.c_str(), '/');
fileObj["name"] = name ? name + 1 : file.path.c_str();
// Stat the file for size
FsFile f;
if (Storage.openFileForRead("WEB", file.path.c_str(), f)) {
fileObj["size"] = static_cast<unsigned long>(f.size());
f.close();
} else {
fileObj["size"] = 0;
}
}
}
String json;
serializeJson(doc, json);
server->send(200, "application/json", json);
}
void CrossPointWebServer::handleFontUploadData() {
HTTPUpload& upload = server->upload();
switch (upload.status) {
case UPLOAD_FILE_START: {
esp_task_wdt_reset();
String family = server->arg("family");
fontUpload.valid = false;
fontUpload.magicChecked = false;
fontUpload.bytesWritten = 0;
fontUpload.bufferPos = 0;
if (!FontInstaller::isValidFamilyName(family.c_str())) {
LOG_ERR("WEB", "Invalid font family name: %s", family.c_str());
break;
}
String filename = upload.filename;
// Validate filename: rejects path traversal (../, /, \) and enforces
// a .cpfont basename of alphanumeric + hyphen + underscore. Without
// this an attacker could supply "../../.crosspoint/settings.json" as
// a "filename" and have it written outside the fonts directory.
if (!FontInstaller::isValidCpfontFilename(filename.c_str())) {
LOG_ERR("WEB", "Invalid font filename: %s", filename.c_str());
break;
}
fontUpload.familyName = family.c_str();
// Create a temporary FontInstaller for directory creation
FontInstaller installer(sdFontSystem.registry());
if (!installer.ensureFamilyDir(family.c_str())) {
LOG_ERR("WEB", "Failed to create font family dir");
break;
}
char path[128];
FontInstaller::buildFontPath(family.c_str(), filename.c_str(), path, sizeof(path));
fontUpload.filePath = path;
if (!Storage.openFileForWrite("WEB", path, fontUpload.file)) {
LOG_ERR("WEB", "Failed to open font file for write: %s", path);
break;
}
fontUpload.valid = true;
LOG_DBG("WEB", "Font upload started: %s -> %s", filename.c_str(), path);
break;
}
case UPLOAD_FILE_WRITE: {
if (!fontUpload.valid) break;
esp_task_wdt_reset();
// Validate magic bytes on first chunk only
if (!fontUpload.magicChecked && upload.currentSize >= 8) {
if (memcmp(upload.buf, "CPFONT\0\0", 8) != 0) {
LOG_ERR("WEB", "Invalid .cpfont magic bytes");
fontUpload.valid = false;
break;
}
fontUpload.magicChecked = true;
}
// Buffer writes for efficiency
size_t remaining = upload.currentSize;
const uint8_t* src = upload.buf;
while (remaining > 0) {
size_t space = FontUploadState::BUFFER_SIZE - fontUpload.bufferPos;
size_t chunk = (remaining < space) ? remaining : space;
memcpy(fontUpload.buffer.data() + fontUpload.bufferPos, src, chunk);
fontUpload.bufferPos += chunk;
src += chunk;
remaining -= chunk;
if (fontUpload.bufferPos >= FontUploadState::BUFFER_SIZE) {
fontUpload.file.write(fontUpload.buffer.data(), fontUpload.bufferPos);
fontUpload.bytesWritten += fontUpload.bufferPos;
fontUpload.bufferPos = 0;
esp_task_wdt_reset();
}
}
break;
}
case UPLOAD_FILE_END: {
// Flush remaining buffer
if (fontUpload.valid && fontUpload.bufferPos > 0) {
fontUpload.file.write(fontUpload.buffer.data(), fontUpload.bufferPos);
fontUpload.bytesWritten += fontUpload.bufferPos;
fontUpload.bufferPos = 0;
}
fontUpload.file.close();
if (!fontUpload.valid && !fontUpload.filePath.empty()) {
Storage.remove(fontUpload.filePath.c_str());
}
LOG_DBG("WEB", "Font upload end: valid=%d, %zu bytes", fontUpload.valid, fontUpload.bytesWritten);
break;
}
case UPLOAD_FILE_ABORTED: {
fontUpload.file.close();
if (!fontUpload.filePath.empty()) {
Storage.remove(fontUpload.filePath.c_str());
}
fontUpload.valid = false;
LOG_DBG("WEB", "Font upload aborted");
break;
}
}
}
void CrossPointWebServer::handleFontUpload() {
if (fontUpload.valid) {
sdFontSystem.markRegistryDirty();
server->send(200, "application/json", "{\"ok\":true}");
LOG_DBG("WEB", "Font upload complete: %s", fontUpload.filePath.c_str());
} else {
server->send(400, "application/json", "{\"error\":\"Invalid .cpfont file\"}");
}
}
void CrossPointWebServer::handleFontDelete() {
String body = server->arg("plain");
JsonDocument doc;
DeserializationError err = deserializeJson(doc, body);
if (err || !doc["family"].is<const char*>()) {
server->send(400, "application/json", "{\"error\":\"Invalid request\"}");
return;
}
const char* familyName = doc["family"];
FontInstaller installer(sdFontSystem.registry());
auto result = installer.deleteFamily(familyName);
if (result == FontInstaller::Error::OK) {
sdFontSystem.markRegistryDirty();
server->send(200, "application/json", "{\"ok\":true}");
LOG_DBG("WEB", "Deleted font family: %s", familyName);
} else {
server->send(500, "application/json", "{\"error\":\"Delete failed\"}");
LOG_ERR("WEB", "Failed to delete font family: %s", familyName);
}
}
+22
View File
@@ -108,6 +108,28 @@ class CrossPointWebServer {
void handleGetSettings() const;
void handlePostSettings();
// Font management handlers
void handleFontsPage() const;
void handleFontList() const;
void handleFontUpload();
void handleFontUploadData();
void handleFontDelete();
// Font upload state
struct FontUploadState {
FsFile file;
std::string familyName;
std::string filePath;
bool valid = false;
bool magicChecked = false;
size_t bytesWritten = 0;
static constexpr size_t BUFFER_SIZE = 4096;
std::vector<uint8_t> buffer;
size_t bufferPos = 0;
FontUploadState() { buffer.resize(BUFFER_SIZE); }
} fontUpload;
// OPDS server handlers
void handleGetOpdsServers() const;
void handlePostOpdsServer();
+1
View File
@@ -1465,6 +1465,7 @@
<a href="/">Home</a>
<a href="/files" class="active">File Manager</a>
<a href="/settings">Settings</a>
<a href="/fonts">Fonts</a>
</div>
<div class="page-header">
+323
View File
@@ -0,0 +1,323 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>CrossPoint Reader - Fonts</title>
<style>
:root {
--font-color: #333;
--bg: #f5f5f5;
--title-color: #2c3e50;
--card-bg: #FFF;
--label-color: #7f8c8d;
--border-color: #eee;
--accent-color: rgb(110, 154, 130);
--accent-hover-color: #5a8c73;
--danger-color: #e74c3c;
--danger-hover: #c0392b;
}
@media (prefers-color-scheme: dark) {
:root {
--font-color: #f5f5f5;
--bg: #333;
--title-color: #ecf0f1;
--card-bg: #444;
--label-color: #bdc3c7;
--border-color: #555;
color-scheme: dark;
}
}
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
Oxygen, Ubuntu, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
background-color: var(--bg);
color: var(--font-color);
}
h1 {
color: var(--title-color);
border-bottom: 2px solid var(--accent-color);
padding-bottom: 10px;
}
h2 { color: var(--title-color); margin-top: 0; }
h3 { margin: 0 0 8px 0; }
.card {
background: var(--card-bg);
border-radius: 8px;
padding: 20px;
margin: 15px 0;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
}
.nav-links {
margin: 20px 0;
display: flex;
gap: 10px;
}
.nav-links a {
padding: 10px 20px;
color: var(--font-color);
text-decoration: none;
border-radius: 4px;
}
.nav-links a.active {
background-color: var(--accent-color);
color: white;
}
.nav-links a:not(.active):hover {
background-color: var(--accent-hover-color);
color: white;
}
.family {
border-bottom: 1px solid var(--border-color);
padding: 12px 0;
display: flex;
justify-content: space-between;
align-items: center;
}
.family:last-child { border-bottom: none; }
.family-info { flex: 1; }
.family-meta { color: var(--label-color); font-size: 0.9em; }
.btn {
padding: 6px 16px;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 0.9em;
}
.btn-danger {
background: var(--danger-color);
color: white;
}
.btn-danger:hover { background: var(--danger-hover); }
.btn-primary {
background: var(--accent-color);
color: white;
}
.btn-primary:hover { background: var(--accent-hover-color); }
.upload-form {
display: flex;
gap: 10px;
align-items: center;
flex-wrap: wrap;
}
.upload-form input[type="text"] {
padding: 6px 10px;
border: 1px solid var(--border-color);
border-radius: 4px;
background: var(--bg);
color: var(--font-color);
}
.upload-form input[type="file"] { flex: 1; min-width: 200px; }
#status {
margin-top: 10px;
padding: 8px;
border-radius: 4px;
display: none;
}
.status-ok { background: #d4edda; color: #155724; display: block !important; }
.status-err { background: #f8d7da; color: #721c24; display: block !important; }
.empty { color: var(--label-color); text-align: center; padding: 20px; }
</style>
</head>
<body>
<h1>📚 CrossPoint Reader</h1>
<div class="nav-links">
<a href="/">Home</a>
<a href="/files">File Manager</a>
<a href="/settings">Settings</a>
<a href="/fonts" class="active">Fonts</a>
</div>
<div class="card">
<h2>Installed Fonts</h2>
<div id="families"><p class="empty">Loading...</p></div>
</div>
<div class="card">
<h2>Upload Font</h2>
<form class="upload-form" id="uploadForm">
<input type="file" id="fontFiles" webkitdirectory directory multiple required />
<button type="submit" class="btn btn-primary">Upload</button>
</form>
<p id="pickedInfo" class="family-meta" style="margin: 8px 0 0;"></p>
<div id="status"></div>
</div>
<script>
function formatSize(bytes) {
if (bytes >= 1048576) return (bytes / 1048576).toFixed(1) + ' MB';
if (bytes >= 1024) return (bytes / 1024).toFixed(0) + ' KB';
return bytes + ' B';
}
async function loadFonts() {
const el = document.getElementById('families');
try {
const res = await fetch('/api/fonts');
const data = await res.json();
// Build rows with DOM APIs and textContent so on-device family names
// (which can contain arbitrary characters) cannot break markup or
// execute script via innerHTML / inline onclick interpolation.
el.replaceChildren();
if (!data.families || data.families.length === 0) {
const p = document.createElement('p');
p.className = 'empty';
p.textContent = 'No fonts installed';
el.appendChild(p);
return;
}
for (const f of data.families) {
const row = document.createElement('div');
row.className = 'family';
const info = document.createElement('div');
info.className = 'family-info';
const h3 = document.createElement('h3');
h3.textContent = f.name;
info.appendChild(h3);
const meta = document.createElement('span');
meta.className = 'family-meta';
const sizes = (f.sizes || []).join(', ');
const filesSizes = (f.files || []).map(fi => formatSize(fi.size)).join(' + ');
meta.textContent = sizes + 'pt · ' + filesSizes;
info.appendChild(meta);
const btn = document.createElement('button');
btn.className = 'btn btn-danger';
btn.textContent = 'Delete';
// Capture name in the closure rather than interpolating into onclick.
const familyName = f.name;
btn.addEventListener('click', () => deleteFamily(familyName));
row.appendChild(info);
row.appendChild(btn);
el.appendChild(row);
}
} catch (e) {
el.replaceChildren();
const p = document.createElement('p');
p.className = 'empty';
p.textContent = 'Failed to load font list';
el.appendChild(p);
}
}
async function deleteFamily(name) {
if (!confirm('Delete font family "' + name + '"?')) return;
const status = document.getElementById('status');
status.className = '';
status.style.display = 'block';
status.textContent = 'Deleting ' + name + '...';
try {
const res = await fetch('/api/fonts/delete', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({family: name})
});
if (res.ok) {
status.className = 'status-ok';
status.textContent = 'Deleted "' + name + '".';
} else {
status.className = 'status-err';
status.textContent = 'Failed to delete "' + name + '".';
}
} catch (err) {
status.className = 'status-err';
status.textContent = 'Delete error: ' + err.message;
}
await loadFonts();
}
// Derive family name from a .cpfont filename: take everything before the
// last '-' or '_' (that separator precedes the size suffix, e.g. Bookerly_12.cpfont).
function familyFromFilename(name) {
const stem = name.replace(/\.cpfont$/i, '');
const cut = Math.max(stem.lastIndexOf('-'), stem.lastIndexOf('_'));
return cut > 0 ? stem.slice(0, cut) : stem;
}
// Sanitize to match firmware's [A-Za-z0-9_-]+ pattern.
function sanitizeFamily(raw) {
return raw.replace(/[^A-Za-z0-9_-]/g, '_');
}
function cpfontFilesOnly(fileList) {
return Array.from(fileList).filter(f => /\.cpfont$/i.test(f.name));
}
document.getElementById('fontFiles').addEventListener('change', function() {
const info = document.getElementById('pickedInfo');
const files = cpfontFilesOnly(this.files);
if (files.length === 0) {
info.textContent = 'No .cpfont files found in the selected folder.';
return;
}
const family = sanitizeFamily(familyFromFilename(files[0].name));
info.textContent = files.length + ' file' + (files.length === 1 ? '' : 's') +
' → family "' + family + '"';
});
document.getElementById('uploadForm').addEventListener('submit', async function(e) {
e.preventDefault();
const status = document.getElementById('status');
const files = cpfontFilesOnly(document.getElementById('fontFiles').files);
if (files.length === 0) {
status.className = 'status-err';
status.style.display = 'block';
status.textContent = 'No .cpfont files selected.';
return;
}
// A directory picker may include files from multiple family subfolders.
// Reject that up front — otherwise files[0]'s family is silently reused
// for every upload, corrupting the install layout.
const families = [...new Set(files.map(f => sanitizeFamily(familyFromFilename(f.name))))];
if (families.length !== 1) {
status.className = 'status-err';
status.style.display = 'block';
status.textContent = 'Please select files from a single font family.';
return;
}
const family = families[0];
status.className = '';
status.style.display = 'block';
let uploaded = 0;
for (const file of files) {
status.textContent = 'Uploading ' + (uploaded + 1) + '/' + files.length + ': ' + file.name;
const formData = new FormData();
formData.append('family', family);
formData.append('file', file, file.name);
try {
const res = await fetch('/api/fonts/upload', { method: 'POST', body: formData });
const data = await res.json();
if (!data.ok) {
status.className = 'status-err';
status.textContent = 'Failed on ' + file.name + ': ' + (data.error || 'unknown error');
await loadFonts();
return;
}
} catch (err) {
status.className = 'status-err';
status.textContent = 'Upload error on ' + file.name + ': ' + err.message;
await loadFonts();
return;
}
uploaded++;
}
status.className = 'status-ok';
status.textContent = 'Uploaded ' + uploaded + ' file' + (uploaded === 1 ? '' : 's') +
' to family "' + family + '".';
await loadFonts();
});
loadFonts();
</script>
</body>
</html>
+1
View File
@@ -104,6 +104,7 @@
<a href="/" class="active">Home</a>
<a href="/files">File Manager</a>
<a href="/settings">Settings</a>
<a href="/fonts">Fonts</a>
</div>
<div class="card">
+1
View File
@@ -285,6 +285,7 @@
<a href="/">Home</a>
<a href="/files">File Manager</a>
<a href="/settings" class="active">Settings</a>
<a href="/fonts">Fonts</a>
</div>
<div id="message" class="message"></div>