Account for deduplication

lib/I18n/I18n.cpp

@@ -1,7 +1,7 @@
 #include "I18n.h"
 
 #include <HalStorage.h>
-#include <HardwareSerial.h>
+#include <Logging.h>
 #include <Serialization.h>
 
 #include <string>
@@ -27,7 +27,12 @@ const char* I18n::get(StrId id) const {
 
   // Use generated helper function - no hardcoded switch needed!
   const LangStrings lang = getLanguageStrings(_language);
-  return lang.data + lang.offsets[index];
+  const char* result = lang.data + lang.offsets[index];
+  if (_language != Language::EN && result[0] == '\0') {
+    const LangStrings english = getLanguageStrings(Language::EN);
+    return english.data + english.offsets[index];
+  }
+  return result;
 }
 
 void I18n::setLanguage(Language lang) {
@@ -59,7 +64,7 @@ void I18n::saveSettings() {
 
   FsFile file;
   if (!Storage.openFileForWrite("I18N", SETTINGS_FILE, file)) {
-    Serial.printf("[I18N] Failed to save settings\n");
+    LOG_ERR("I18N", "Failed to save settings");
     return;
   }
 
@@ -67,14 +72,13 @@ void I18n::saveSettings() {
   serialization::writeString(file, getLanguageCode(_language));
 
   file.close();
-  Serial.printf("[I18N] Settings saved: language=%d code=%s\n", static_cast<int>(_language),
+  LOG_INF("I18N", "Settings saved: language=%d code=%s", static_cast<int>(_language), getLanguageCode(_language));
-                getLanguageCode(_language));
 }
 
 void I18n::loadSettings() {
   FsFile file;
   if (!Storage.openFileForRead("I18N", SETTINGS_FILE, file)) {
-    Serial.printf("[I18N] No settings file, using default (English)\n");
+    LOG_INF("I18N", "No settings file, using default (English)");
     return;
   }
 
@@ -95,9 +99,9 @@ void I18n::loadSettings() {
     }
 
     if (found) {
-      Serial.printf("[I18N] Loaded language code: %s (%d)\n", code.c_str(), static_cast<int>(_language));
+      LOG_INF("I18N", "Loaded language code: %s (%d)", code.c_str(), static_cast<int>(_language));
     } else {
-      Serial.printf("[I18N] Unknown language code in settings: %s\n", code.c_str());
+      LOG_ERR("I18N", "Unknown language code in settings: %s", code.c_str());
     }
     file.close();
     return;
@@ -109,18 +113,17 @@ void I18n::loadSettings() {
     serialization::readPod(file, lang);
     if (lang < static_cast<size_t>(Language::_COUNT)) {
       _language = static_cast<Language>(lang);
-      Serial.printf("[I18N] Migrating v1 language index: %d -> %s\n", static_cast<int>(_language),
+      LOG_INF("I18N", "Migrating v1 language index: %d -> %s", static_cast<int>(_language), getLanguageCode(_language));
-                    getLanguageCode(_language));
       file.close();
       saveSettings();
       return;
     }
     file.close();
-    Serial.printf("[I18N] Invalid v1 language index: %d\n", static_cast<int>(lang));
+    LOG_ERR("I18N", "Invalid v1 language index: %d", static_cast<int>(lang));
     return;
   }
 
-  Serial.printf("[I18N] Settings version mismatch: %d\n", static_cast<int>(version));
+  LOG_ERR("I18N", "Settings version mismatch: %d\n", static_cast<int>(version));
 
   file.close();
 }

scripts/gen_i18n.py

@@ -206,21 +206,36 @@ def load_translations(
         if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", key):
             raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
 
-    # Build translations dict, filling missing keys from English
+    # Build translations dict, filling missing keys from English.
+    # Non-English values that are absent or blank fall back to English at runtime.
+    # A literal empty string in a non-English YAML file is treated as an intentional
+    # empty translation and stored as a single space to distinguish it from fallback.
     inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
     translations: Dict[str, List[str]] = {}
     for key in string_keys:
         row: List[str] = []
         for lang_idx, fname in enumerate(ordered_files):
             data = parsed[fname]
-            value = data.get(key, "")
+            if key not in data:
-            if not value.strip() and fname != english_file:
+                value = ""
-                value = english_data[key]
+                if fname != english_file:
-                inherited_sets[lang_idx].add(key)
+                    inherited_sets[lang_idx].add(key)
-                if verbose:
+                    if verbose:
-                    print(
+                        print(
-                        f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
+                            f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
-                    )
+                        )
+            else:
+                value = data[key]
+                if fname != english_file:
+                    if value == "":
+                        value = " "
+                    elif not value.strip():
+                        value = ""
+                        inherited_sets[lang_idx].add(key)
+                        if verbose:
+                            print(
+                                f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
+                            )
             row.append(value)
         translations[key] = row
 
@@ -474,8 +489,12 @@ def format_cpp_string_literal(segments: List[str], indent: str = "    ") -> List
 def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str:
     """Return a sorted string of every unique character used in a language."""
     chars = set()
+    english_index = 0
     for values in translations.values():
-        for ch in values[lang_index]:
+        text = values[lang_index]
+        if lang_index != english_index and text == "":
+            text = values[english_index]
+        for ch in text:
             chars.add(ord(ch))
     return "".join(chr(cp) for cp in sorted(chars))
 
@@ -741,20 +760,43 @@ def _print_language_table(
     string_keys: List[str],
     unused_keys: Set[str],
     data_sizes: List[int],
+    translations: Dict[str, List[str]],
 ) -> None:
     """Print a per-language summary table."""
     total = len(string_keys)
-    headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)")
+    headers = (
+        "Language",
+        "Code",
+        "Own",
+        "Fallback",
+        "Unused",
+        "Data (B)",
+        "Unique (B)",
+    )
 
     rows = []
-    for code, name, inherited, size in zip(
+    for lang_idx, (code, name, inherited, size) in enumerate(
-        language_codes, language_names, inherited_sets, data_sizes
+        zip(language_codes, language_names, inherited_sets, data_sizes)
     ):
         own = total - len(inherited)
         fallback = len(inherited)
         # strings this language translated but the code never calls
         unused = len(unused_keys - inherited)
-        rows.append((name, code, str(own), str(fallback), str(unused), str(size)))
+        unique_size = sum(
+            len(s.encode("utf-8")) + 1
+            for s in {translations[k][lang_idx] for k in string_keys}
+        )
+        rows.append(
+            (
+                name,
+                code,
+                str(own),
+                str(fallback),
+                str(unused),
+                str(size),
+                str(unique_size),
+            )
+        )
 
     # EN first, then alphabetically by ISO code
     rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
@@ -785,10 +827,16 @@ def _print_language_table(
     # Current layout: uint16_t offset table (2 B per string per language)
     offset_table_size = n_lang * n_keys * 2
     current_total = total_size + offset_table_size
-    # Previous layout: const char* pointer array (4 B per string per language)
+
+    # Estimate the original pointer-based layout using deduplicated string storage.
+    unique_strings: Set[str] = set()
+    for values in translations.values():
+        unique_strings.update(values)
+    unique_string_size = sum(len(s.encode("utf-8")) + 1 for s in unique_strings)
     old_pointer_table_size = n_lang * n_keys * 4
-    old_total = total_size + old_pointer_table_size
+    old_total = unique_string_size + old_pointer_table_size
     saved = old_total - current_total
+
     print(
         f"\n  Total: {total}  |  Used in code: {used}  |  Never used: {len(unused_keys)}"
     )
@@ -797,10 +845,10 @@ def _print_language_table(
         f"  =  {current_total:>7,} B"
     )
     print(
-        f"  Flash (before): {total_size:>7,} B strings  +  {old_pointer_table_size:>6,} B pointer tables (ptr32)"
+        f"  Flash (pointer model, deduped): {unique_string_size:>7,} B unique strings  +  {old_pointer_table_size:>6,} B pointer tables (ptr32)"
         f"  =  {old_total:>7,} B"
     )
-    print(f"  Saved by offset tables: {saved:,} B")
+    print(f"  Estimated savings vs pointer model: {saved:,} B")
 
 
 def _append_string_data_entry(lines: List[str], text: str) -> None:
@@ -917,6 +965,7 @@ def main(
             string_keys,
             unused_set,
             data_sizes,
+            translations,
         )
         print()