Merge pull request #159 from jpirnay/refactor-geni18n-dedup

refactor: Account for deduplication
2026-05-01 09:58:13 +02:00
parent 3fb88193ad 6f4a74120f
commit 0b345bcc6b
3 changed files with 83 additions and 31 deletions
@@ -1,7 +1,7 @@
 #include "I18n.h"

 #include <HalStorage.h>
-#include <HardwareSerial.h>
+#include <Logging.h>
 #include <Serialization.h>

 #include <string>
@@ -27,7 +27,12 @@ const char* I18n::get(StrId id) const {

  // Use generated helper function - no hardcoded switch needed!
  const LangStrings lang = getLanguageStrings(_language);
-  return lang.data + lang.offsets[index];
+  const char* result = lang.data + lang.offsets[index];
+  if (_language != Language::EN && result[0] == '\0') {
+    const LangStrings english = getLanguageStrings(Language::EN);
+    return english.data + english.offsets[index];
+  }
+  return result;
 }

 void I18n::setLanguage(Language lang) {
@@ -59,7 +64,7 @@ void I18n::saveSettings() {

  FsFile file;
  if (!Storage.openFileForWrite("I18N", SETTINGS_FILE, file)) {
-    Serial.printf("[I18N] Failed to save settings\n");
+    LOG_ERR("I18N", "Failed to save settings");
    return;
  }

@@ -67,14 +72,13 @@ void I18n::saveSettings() {
  serialization::writeString(file, getLanguageCode(_language));

  file.close();
-  Serial.printf("[I18N] Settings saved: language=%d code=%s\n", static_cast<int>(_language),
-                getLanguageCode(_language));
+  LOG_INF("I18N", "Settings saved: language=%d code=%s", static_cast<int>(_language), getLanguageCode(_language));
 }

 void I18n::loadSettings() {
  FsFile file;
  if (!Storage.openFileForRead("I18N", SETTINGS_FILE, file)) {
-    Serial.printf("[I18N] No settings file, using default (English)\n");
+    LOG_INF("I18N", "No settings file, using default (English)");
    return;
  }

@@ -95,9 +99,9 @@ void I18n::loadSettings() {
    }

    if (found) {
-      Serial.printf("[I18N] Loaded language code: %s (%d)\n", code.c_str(), static_cast<int>(_language));
+      LOG_INF("I18N", "Loaded language code: %s (%d)", code.c_str(), static_cast<int>(_language));
    } else {
-      Serial.printf("[I18N] Unknown language code in settings: %s\n", code.c_str());
+      LOG_ERR("I18N", "Unknown language code in settings: %s", code.c_str());
    }
    file.close();
    return;
@@ -109,18 +113,17 @@ void I18n::loadSettings() {
    serialization::readPod(file, lang);
    if (lang < static_cast<size_t>(Language::_COUNT)) {
      _language = static_cast<Language>(lang);
-      Serial.printf("[I18N] Migrating v1 language index: %d -> %s\n", static_cast<int>(_language),
-                    getLanguageCode(_language));
+      LOG_INF("I18N", "Migrating v1 language index: %d -> %s", static_cast<int>(_language), getLanguageCode(_language));
      file.close();
      saveSettings();
      return;
    }
    file.close();
-    Serial.printf("[I18N] Invalid v1 language index: %d\n", static_cast<int>(lang));
+    LOG_ERR("I18N", "Invalid v1 language index: %d", static_cast<int>(lang));
    return;
  }

-  Serial.printf("[I18N] Settings version mismatch: %d\n", static_cast<int>(version));
+  LOG_ERR("I18N", "Settings version mismatch: %d\n", static_cast<int>(version));

  file.close();
 }
@@ -206,21 +206,36 @@ def load_translations(
        if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", key):
            raise ValueError(f"Invalid C++ identifier in English file: '{key}'")

-    # Build translations dict, filling missing keys from English
+    # Build translations dict, filling missing keys from English.
+    # Non-English values that are absent or blank fall back to English at runtime.
+    # A literal empty string in a non-English YAML file is treated as an intentional
+    # empty translation and stored as a single space to distinguish it from fallback.
    inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
    translations: Dict[str, List[str]] = {}
    for key in string_keys:
        row: List[str] = []
        for lang_idx, fname in enumerate(ordered_files):
            data = parsed[fname]
-            value = data.get(key, "")
-            if not value.strip() and fname != english_file:
-                value = english_data[key]
-                inherited_sets[lang_idx].add(key)
-                if verbose:
-                    print(
-                        f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
-                    )
+            if key not in data:
+                value = ""
+                if fname != english_file:
+                    inherited_sets[lang_idx].add(key)
+                    if verbose:
+                        print(
+                            f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
+                        )
+            else:
+                value = data[key]
+                if fname != english_file:
+                    if value == "":
+                        value = " "
+                    elif not value.strip():
+                        value = ""
+                        inherited_sets[lang_idx].add(key)
+                        if verbose:
+                            print(
+                                f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
+                            )
            row.append(value)
        translations[key] = row

@@ -474,8 +489,12 @@ def format_cpp_string_literal(segments: List[str], indent: str = "    ") -> List
 def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str:
    """Return a sorted string of every unique character used in a language."""
    chars = set()
+    english_index = 0
    for values in translations.values():
-        for ch in values[lang_index]:
+        text = values[lang_index]
+        if lang_index != english_index and text == "":
+            text = values[english_index]
+        for ch in text:
            chars.add(ord(ch))
    return "".join(chr(cp) for cp in sorted(chars))

@@ -741,20 +760,43 @@ def _print_language_table(
    string_keys: List[str],
    unused_keys: Set[str],
    data_sizes: List[int],
+    translations: Dict[str, List[str]],
 ) -> None:
    """Print a per-language summary table."""
    total = len(string_keys)
-    headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)")
+    headers = (
+        "Language",
+        "Code",
+        "Own",
+        "Fallback",
+        "Unused",
+        "Data (B)",
+        "Unique (B)",
+    )

    rows = []
-    for code, name, inherited, size in zip(
-        language_codes, language_names, inherited_sets, data_sizes
+    for lang_idx, (code, name, inherited, size) in enumerate(
+        zip(language_codes, language_names, inherited_sets, data_sizes)
    ):
        own = total - len(inherited)
        fallback = len(inherited)
        # strings this language translated but the code never calls
        unused = len(unused_keys - inherited)
-        rows.append((name, code, str(own), str(fallback), str(unused), str(size)))
+        unique_size = sum(
+            len(s.encode("utf-8")) + 1
+            for s in {translations[k][lang_idx] for k in string_keys}
+        )
+        rows.append(
+            (
+                name,
+                code,
+                str(own),
+                str(fallback),
+                str(unused),
+                str(size),
+                str(unique_size),
+            )
+        )

    # EN first, then alphabetically by ISO code
    rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
@@ -785,10 +827,16 @@ def _print_language_table(
    # Current layout: uint16_t offset table (2 B per string per language)
    offset_table_size = n_lang * n_keys * 2
    current_total = total_size + offset_table_size
-    # Previous layout: const char* pointer array (4 B per string per language)
+
+    # Estimate the original pointer-based layout using deduplicated string storage.
+    unique_strings: Set[str] = set()
+    for values in translations.values():
+        unique_strings.update(values)
+    unique_string_size = sum(len(s.encode("utf-8")) + 1 for s in unique_strings)
    old_pointer_table_size = n_lang * n_keys * 4
-    old_total = total_size + old_pointer_table_size
+    old_total = unique_string_size + old_pointer_table_size
    saved = old_total - current_total
+
    print(
        f"\n  Total: {total}  |  Used in code: {used}  |  Never used: {len(unused_keys)}"
    )
@@ -797,10 +845,10 @@ def _print_language_table(
        f"  =  {current_total:>7,} B"
    )
    print(
-        f"  Flash (before): {total_size:>7,} B strings  +  {old_pointer_table_size:>6,} B pointer tables (ptr32)"
+        f"  Flash (pointer model, deduped): {unique_string_size:>7,} B unique strings  +  {old_pointer_table_size:>6,} B pointer tables (ptr32)"
        f"  =  {old_total:>7,} B"
    )
-    print(f"  Saved by offset tables: {saved:,} B")
+    print(f"  Estimated savings vs pointer model: {saved:,} B")


 def _append_string_data_entry(lines: List[str], text: str) -> None:
@@ -917,6 +965,7 @@ def main(
            string_keys,
            unused_set,
            data_sizes,
+            translations,
        )
        print()

@@ -25,7 +25,7 @@ class SleepActivity final : public Activity {
  void renderCustomSleepScreen() const;
  void renderCoverSleepScreen() const;
  void renderBitmapSleepScreen(const Bitmap& bitmap, const BookOverlayInfo& overlayInfo,
-                                bool topAlignForCoverFit = false) const;
+                               bool topAlignForCoverFit = false) const;
  void renderBlankSleepScreen() const;
  void renderOverlaySleepScreen() const;
  BookOverlayInfo getBookOverlayInfo(const std::string& bookPath) const;