feat: add SD card font support with on-device download and web management

Add a complete SD card font subsystem that enables users to install and use custom fonts beyond the three built-in families. This combines the back-end firmware support (#1327) with the font configuration, build pipeline, CI distribution, and user-facing management UI (#1392). Core font system: - Custom .cpfont binary format (v4) with multi-style support (regular, bold, italic, bold-italic) packed into a single file per size - On-demand glyph loading from SD card with two-pass prewarm rendering to bulk-read glyphs per page, achieving near-flash performance for Latin text (~697ms vs ~681ms) and viable CJK rendering (~32% slower) - Persistent advance cache for layout measurement without SD I/O - Overflow ring buffer for glyph cache misses during rendering - Memory-conscious design: only advance tables kept in RAM; glyph bitmaps, kern tables, and ligatures loaded on demand from SD Font management: - On-device WiFi download from GitHub Releases with manifest-based discovery, install/update detection, and progress UI - Web interface font upload, listing, and deletion via /fonts page - Manual SD card copy to /fonts/ or /.fonts/ directories - Font selection integrated into Settings > Reader > Font Family Build pipeline: - Declarative YAML config (sd-fonts.yaml) as single source of truth for the 17-family font library (serif, sans, mono, accessibility) - Python converter (fontconvert_sdcard.py) for TTF/OTF to .cpfont with FreeType rasterization, class-based kerning, and ligature extraction - Parallel build orchestrator with variable font instance extraction - CI workflow publishing versioned + stable releases to a dedicated crosspoint-fonts repository with auto-incrementing revision tags - Centralized version constants (cpfont_version.py) shared across build tooling and CI, with firmware headers as manual sync points Additional fixes: - CJK characters no longer get hyphens inserted at line breaks - Advance table eliminates 30+ second stalls during CJK section indexing for paragraphs with >512 unique codepoints Closes #930 Co-authored-by: Zach Nelson <zach@zdnelson.com> Co-authored-by: Justin <itsthisjustin@users.noreply.github.com> Co-authored-by: jpirnay <jens@pirnay.com> Co-authored-by: mcrosson <kemonine@kemonine.info>
2026-05-08 21:50:06 -05:00
parent 29fd29f537
commit 7993b2bb97
57 changed files with 6064 additions and 54 deletions
@@ -0,0 +1,426 @@
+#include "FontDownloadActivity.h"
+
+#include <ArduinoJson.h>
+#include <GfxRenderer.h>
+#include <HalStorage.h>
+#include <I18n.h>
+#include <Logging.h>
+#include <WiFi.h>
+
+#include "MappedInputManager.h"
+#include "SdCardFontGlobals.h"
+#include "activities/network/WifiSelectionActivity.h"
+#include "components/UITheme.h"
+#include "fontIds.h"
+#include "network/HttpDownloader.h"
+
+FontDownloadActivity::FontDownloadActivity(GfxRenderer& renderer, MappedInputManager& mappedInput)
+    : Activity("FontDownload", renderer, mappedInput), fontInstaller_(sdFontSystem.registry()) {}
+
+// --- Lifecycle ---
+
+void FontDownloadActivity::onEnter() {
+  Activity::onEnter();
+  WiFi.mode(WIFI_STA);
+  startActivityForResult(std::make_unique<WifiSelectionActivity>(renderer, mappedInput),
+                         [this](const ActivityResult& result) { onWifiSelectionComplete(!result.isCancelled); });
+}
+
+void FontDownloadActivity::onExit() {
+  Activity::onExit();
+  WiFi.disconnect(false);
+  delay(100);
+  WiFi.mode(WIFI_OFF);
+  delay(100);
+}
+
+void FontDownloadActivity::onWifiSelectionComplete(const bool success) {
+  if (!success) {
+    finish();
+    return;
+  }
+
+  {
+    RenderLock lock(*this);
+    state_ = LOADING_MANIFEST;
+  }
+  requestUpdateAndWait();
+
+  if (!fetchAndParseManifest()) {
+    {
+      RenderLock lock(*this);
+      state_ = ERROR;
+    }
+    return;
+  }
+
+  {
+    RenderLock lock(*this);
+    state_ = FAMILY_LIST;
+    selectedIndex_ = 0;
+  }
+}
+
+// --- Manifest fetching ---
+
+bool FontDownloadActivity::fetchAndParseManifest() {
+  // Download manifest to a temp file on SD card to avoid holding both
+  // TLS buffers and the full JSON string in RAM simultaneously.
+  static constexpr const char* MANIFEST_TMP = "/fonts_manifest.tmp";
+
+  auto result = HttpDownloader::downloadToFile(FONT_MANIFEST_URL, MANIFEST_TMP, nullptr);
+  if (result != HttpDownloader::OK) {
+    LOG_ERR("FONT", "Failed to fetch manifest from %s", FONT_MANIFEST_URL);
+    errorMessage_ = "Failed to fetch font list";
+    Storage.remove(MANIFEST_TMP);
+    return false;
+  }
+
+  // HTTP client is now closed — TLS buffers freed. Parse JSON from file.
+  FsFile manifestFile;
+  if (!Storage.openFileForRead("FONT", MANIFEST_TMP, manifestFile)) {
+    LOG_ERR("FONT", "Failed to open temp manifest");
+    Storage.remove(MANIFEST_TMP);
+    errorMessage_ = "Failed to read font list";
+    return false;
+  }
+
+  JsonDocument doc;
+  DeserializationError err = deserializeJson(doc, manifestFile);
+  manifestFile.close();
+  Storage.remove(MANIFEST_TMP);
+
+  if (err) {
+    LOG_ERR("FONT", "Manifest parse error: %s", err.c_str());
+    errorMessage_ = "Invalid font manifest";
+    return false;
+  }
+
+  int version = doc["version"] | 0;
+  if (version != FONTS_MANIFEST_VERSION) {
+    LOG_ERR("FONT", "Unsupported manifest version: %d", version);
+    errorMessage_ = "Unsupported manifest version";
+    return false;
+  }
+
+  baseUrl_ = doc["baseUrl"] | "";
+  families_.clear();
+
+  JsonArray familiesArr = doc["families"].as<JsonArray>();
+  families_.reserve(familiesArr.size());
+
+  for (JsonObject fObj : familiesArr) {
+    ManifestFamily family;
+    family.name = fObj["name"] | "";
+    family.description = fObj["description"] | "";
+
+    for (JsonVariant s : fObj["styles"].as<JsonArray>()) {
+      family.styles.push_back(s.as<std::string>());
+    }
+
+    family.totalSize = 0;
+    for (JsonObject fileObj : fObj["files"].as<JsonArray>()) {
+      ManifestFile file;
+      file.name = fileObj["name"] | "";
+      file.size = fileObj["size"] | 0;
+      family.totalSize += file.size;
+      family.files.push_back(std::move(file));
+    }
+
+    family.installed = fontInstaller_.isFamilyInstalled(family.name.c_str());
+
+    // Detect updates by comparing manifest file sizes with files on disk.
+    // Not a checksum, but a size mismatch reliably indicates a rebuild in practice.
+    if (family.installed) {
+      for (const auto& file : family.files) {
+        char path[128];
+        FontInstaller::buildFontPath(family.name.c_str(), file.name.c_str(), path, sizeof(path));
+        FsFile f;
+        if (Storage.openFileForRead("FONT", path, f)) {
+          size_t actual = f.fileSize();
+          f.close();
+          if (actual != file.size) {
+            family.hasUpdate = true;
+            break;
+          }
+        } else {
+          // File missing on disk but family dir exists — treat as update
+          family.hasUpdate = true;
+          break;
+        }
+      }
+    }
+
+    families_.push_back(std::move(family));
+  }
+
+  LOG_DBG("FONT", "Manifest loaded: %zu families", families_.size());
+  return true;
+}
+
+// --- Download ---
+
+void FontDownloadActivity::downloadAll() {
+  for (size_t i = 0; i < families_.size(); i++) {
+    if (families_[i].installed && !families_[i].hasUpdate) continue;
+    downloadFamily(families_[i]);
+    if (state_ == ERROR) return;
+  }
+
+  {
+    RenderLock lock(*this);
+    state_ = COMPLETE;
+  }
+}
+
+size_t FontDownloadActivity::totalUninstalledSize() const {
+  size_t total = 0;
+  for (const auto& f : families_) {
+    if (!f.installed || f.hasUpdate) total += f.totalSize;
+  }
+  return total;
+}
+
+void FontDownloadActivity::downloadFamily(ManifestFamily& family) {
+  {
+    RenderLock lock(*this);
+    state_ = DOWNLOADING;
+    downloadingFamilyIndex_ = static_cast<int>(&family - families_.data());
+    currentFileIndex_ = 0;
+    currentFileTotal_ = family.files.size();
+    fileProgress_ = 0;
+    fileTotal_ = 0;
+  }
+  requestUpdateAndWait();
+
+  if (!fontInstaller_.ensureFamilyDir(family.name.c_str())) {
+    RenderLock lock(*this);
+    state_ = ERROR;
+    errorMessage_ = "Failed to create font directory";
+    return;
+  }
+
+  for (size_t i = 0; i < family.files.size(); i++) {
+    const auto& file = family.files[i];
+
+    {
+      RenderLock lock(*this);
+      currentFileIndex_ = i;
+      fileProgress_ = 0;
+      fileTotal_ = file.size;
+    }
+    requestUpdateAndWait();
+
+    char destPath[128];
+    FontInstaller::buildFontPath(family.name.c_str(), file.name.c_str(), destPath, sizeof(destPath));
+
+    std::string url = baseUrl_ + file.name;
+
+    auto result = HttpDownloader::downloadToFile(url, destPath, [this](size_t downloaded, size_t total) {
+      fileProgress_ = downloaded;
+      fileTotal_ = total;
+      requestUpdate(true);
+    });
+
+    if (result != HttpDownloader::OK) {
+      LOG_ERR("FONT", "Download failed: %s (%d)", file.name.c_str(), result);
+      fontInstaller_.deleteFamily(family.name.c_str());
+      family.installed = false;
+      family.hasUpdate = false;
+      RenderLock lock(*this);
+      state_ = ERROR;
+      errorMessage_ = "Download failed: " + file.name;
+      return;
+    }
+
+    if (!fontInstaller_.validateCpfontFile(destPath)) {
+      LOG_ERR("FONT", "Invalid .cpfont: %s", destPath);
+      fontInstaller_.deleteFamily(family.name.c_str());
+      family.installed = false;
+      family.hasUpdate = false;
+      RenderLock lock(*this);
+      state_ = ERROR;
+      errorMessage_ = "Invalid font file: " + file.name;
+      return;
+    }
+  }
+
+  fontInstaller_.refreshRegistry();
+  family.installed = true;
+
+  {
+    RenderLock lock(*this);
+    state_ = COMPLETE;
+  }
+}
+
+// --- Input handling ---
+
+void FontDownloadActivity::loop() {
+  if (state_ == FAMILY_LIST) {
+    if (mappedInput.wasPressed(MappedInputManager::Button::Back)) {
+      finish();
+      return;
+    }
+
+    buttonNavigator_.onNextRelease([this] {
+      if (selectedIndex_ < listItemCount() - 1) {
+        selectedIndex_++;
+        requestUpdate();
+      }
+    });
+
+    buttonNavigator_.onPreviousRelease([this] {
+      if (selectedIndex_ > 0) {
+        selectedIndex_--;
+        requestUpdate();
+      }
+    });
+
+    if (mappedInput.wasPressed(MappedInputManager::Button::Confirm)) {
+      if (!families_.empty()) {
+        if (isDownloadAllSelected()) {
+          downloadAll();
+        } else {
+          const auto& family = families_[familyIndexFromList(selectedIndex_)];
+          if (!family.installed || family.hasUpdate) {
+            downloadFamily(families_[familyIndexFromList(selectedIndex_)]);
+          }
+        }
+        requestUpdateAndWait();
+        return;
+      }
+    }
+  } else if (state_ == COMPLETE) {
+    if (mappedInput.wasPressed(MappedInputManager::Button::Back) ||
+        mappedInput.wasPressed(MappedInputManager::Button::Confirm)) {
+      {
+        RenderLock lock(*this);
+        state_ = FAMILY_LIST;
+      }
+      requestUpdate();
+    }
+  } else if (state_ == ERROR) {
+    if (mappedInput.wasPressed(MappedInputManager::Button::Back)) {
+      {
+        RenderLock lock(*this);
+        state_ = FAMILY_LIST;
+      }
+      requestUpdate();
+    } else if (mappedInput.wasPressed(MappedInputManager::Button::Confirm)) {
+      if (downloadingFamilyIndex_ >= 0 && downloadingFamilyIndex_ < static_cast<int>(families_.size())) {
+        downloadFamily(families_[downloadingFamilyIndex_]);
+        requestUpdateAndWait();
+        return;
+      } else {
+        {
+          RenderLock lock(*this);
+          state_ = FAMILY_LIST;
+        }
+        requestUpdate();
+      }
+    }
+  }
+}
+
+// --- Rendering ---
+
+std::string FontDownloadActivity::formatSize(size_t bytes) {
+  char buf[32];
+  if (bytes >= 1024 * 1024) {
+    snprintf(buf, sizeof(buf), "%.1f MB", static_cast<double>(bytes) / (1024.0 * 1024.0));
+  } else if (bytes >= 1024) {
+    snprintf(buf, sizeof(buf), "%.0f KB", static_cast<double>(bytes) / 1024.0);
+  } else {
+    snprintf(buf, sizeof(buf), "%zu B", bytes);
+  }
+  return buf;
+}
+
+void FontDownloadActivity::render(RenderLock&&) {
+  const auto& metrics = UITheme::getInstance().getMetrics();
+  const auto pageWidth = renderer.getScreenWidth();
+  const auto pageHeight = renderer.getScreenHeight();
+
+  renderer.clearScreen();
+
+  GUI.drawHeader(renderer, Rect{0, metrics.topPadding, pageWidth, metrics.headerHeight}, tr(STR_FONT_DOWNLOAD));
+
+  const auto lineHeight = renderer.getLineHeight(UI_10_FONT_ID);
+  const auto contentTop = metrics.topPadding + metrics.headerHeight + metrics.verticalSpacing;
+  const auto centerY = (pageHeight - lineHeight) / 2;
+
+  if (state_ == LOADING_MANIFEST) {
+    renderer.drawCenteredText(UI_10_FONT_ID, centerY, tr(STR_LOADING_FONT_LIST));
+  } else if (state_ == FAMILY_LIST) {
+    if (families_.empty()) {
+      renderer.drawCenteredText(UI_10_FONT_ID, centerY, tr(STR_NO_FONTS_AVAILABLE));
+      const auto labels = mappedInput.mapLabels(tr(STR_BACK), "", "", "");
+      GUI.drawButtonHints(renderer, labels.btn1, labels.btn2, labels.btn3, labels.btn4);
+    } else {
+      GUI.drawList(
+          renderer,
+          Rect{0, contentTop, pageWidth, pageHeight - contentTop - metrics.buttonHintsHeight - metrics.verticalSpacing},
+          listItemCount(), selectedIndex_,
+          [this](int index) -> std::string {
+            if (index == 0) {
+              return std::string(tr(STR_DOWNLOAD_ALL)) + " (" + formatSize(totalUninstalledSize()) + ")";
+            }
+            return families_[familyIndexFromList(index)].name;
+          },
+          nullptr, nullptr,
+          [this](int index) -> std::string {
+            if (index == 0) return "";
+            const auto& f = families_[familyIndexFromList(index)];
+            if (f.hasUpdate) return tr(STR_UPDATE_AVAILABLE);
+            if (f.installed) return tr(STR_INSTALLED);
+            return f.description;
+          },
+          true,
+          [this](int index) -> bool {
+            if (index == 0) return false;
+            const auto& f = families_[familyIndexFromList(index)];
+            // Dim installed fonts, but not those with updates available
+            return f.installed && !f.hasUpdate;
+          });
+
+      const auto labels = mappedInput.mapLabels(tr(STR_BACK), tr(STR_DOWNLOAD), tr(STR_DIR_UP), tr(STR_DIR_DOWN));
+      GUI.drawButtonHints(renderer, labels.btn1, labels.btn2, labels.btn3, labels.btn4);
+    }
+  } else if (state_ == DOWNLOADING) {
+    const auto& family = families_[downloadingFamilyIndex_];
+
+    std::string statusText = std::string(tr(STR_DOWNLOADING)) + " " + family.name + " (" +
+                             std::to_string(currentFileIndex_ + 1) + "/" + std::to_string(currentFileTotal_) + ")";
+    renderer.drawCenteredText(UI_10_FONT_ID, centerY - lineHeight, statusText.c_str());
+
+    float progress = 0;
+    if (fileTotal_ > 0) {
+      progress = static_cast<float>(fileProgress_) / static_cast<float>(fileTotal_);
+    }
+
+    int barY = centerY + metrics.verticalSpacing;
+    GUI.drawProgressBar(
+        renderer,
+        Rect{metrics.contentSidePadding, barY, pageWidth - metrics.contentSidePadding * 2, metrics.progressBarHeight},
+        static_cast<int>(progress * 100), 100);
+
+    int percentY = barY + metrics.progressBarHeight + metrics.verticalSpacing;
+    renderer.drawCenteredText(UI_10_FONT_ID, percentY,
+                              (std::to_string(static_cast<int>(progress * 100)) + "%").c_str());
+  } else if (state_ == COMPLETE) {
+    renderer.drawCenteredText(UI_10_FONT_ID, centerY, tr(STR_FONT_INSTALLED), true, EpdFontFamily::BOLD);
+    const auto labels = mappedInput.mapLabels(tr(STR_BACK), "", "", "");
+    GUI.drawButtonHints(renderer, labels.btn1, labels.btn2, labels.btn3, labels.btn4);
+  } else if (state_ == ERROR) {
+    renderer.drawCenteredText(UI_10_FONT_ID, centerY - lineHeight, tr(STR_FONT_INSTALL_FAILED), true,
+                              EpdFontFamily::BOLD);
+    if (!errorMessage_.empty()) {
+      renderer.drawCenteredText(UI_10_FONT_ID, centerY + metrics.verticalSpacing, errorMessage_.c_str());
+    }
+    const auto labels = mappedInput.mapLabels(tr(STR_BACK), tr(STR_RETRY), "", "");
+    GUI.drawButtonHints(renderer, labels.btn1, labels.btn2, labels.btn3, labels.btn4);
+  }
+
+  renderer.displayBuffer();
+}