Optimize output

lib/I18n/I18n.cpp

@@ -24,8 +24,8 @@ const char* I18n::get(StrId id) const {
   }
 
   // Use generated helper function - no hardcoded switch needed!
-  const char* const* strings = getStringArray(_language);
+  const LangStrings lang = getLanguageStrings(_language);
-  return strings[index];
+  return lang.data + lang.offsets[index];
 }
 
 void I18n::setLanguage(Language lang) {

scripts/gen_i18n.py

@@ -15,24 +15,32 @@ Each YAML file must contain:
 The English file is the reference. Missing keys in other languages are
 automatically filled from English, with a warning.
 
-Usage:
+By default the script scans the src/ and lib/ trees for STR_* references and
-    python gen_i18n.py <translations_dir> <output_dir>
+reports any translation keys that are never used.  Pass --strip-unused to
+omit those keys from the generated output entirely.
 
-Example:
+Usage:
+    python gen_i18n.py [translations_dir [output_dir]] [options]
+
+Examples:
+    python gen_i18n.py
     python gen_i18n.py lib/I18n/translations lib/I18n/
+    python gen_i18n.py --strip-unused
+    python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont
 """
 
 import sys
 import os
 import re
 from pathlib import Path
-from typing import List, Dict, Tuple
+from typing import Dict, List, Optional, Set, Tuple
 
 
 # ---------------------------------------------------------------------------
 # YAML file reading (simple key: "value" format, no PyYAML dependency)
 # ---------------------------------------------------------------------------
 
+
 def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str:
     """
     Process escape sequences in a YAML value string.
@@ -51,9 +59,7 @@ def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str
             elif nxt == "n":
                 result.append("\n")
             else:
-                raise ValueError(
+                raise ValueError(f"{filepath}:{line_num}: unknown escape '\\{nxt}'")
-                    f"{filepath}:{line_num}: unknown escape '\\{nxt}'"
-                )
             i += 2
         else:
             result.append(raw[i])
@@ -103,9 +109,11 @@ def parse_yaml_file(filepath: str) -> Dict[str, str]:
 # Load all languages from a directory of YAML files
 # ---------------------------------------------------------------------------
 
+
 def load_translations(
     translations_dir: str,
-) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]]]:
+    verbose: bool = False,
+) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]:
     """
     Read every YAML file in *translations_dir* and return:
         language_codes   e.g. ["EN", "ES", ...]
@@ -174,16 +182,20 @@ def load_translations(
             raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
 
     # Build translations dict, filling missing keys from English
+    inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
     translations: Dict[str, List[str]] = {}
     for key in string_keys:
         row: List[str] = []
-        for fname in ordered_files:
+        for lang_idx, fname in enumerate(ordered_files):
             data = parsed[fname]
             value = data.get(key, "")
             if not value.strip() and fname != english_file:
                 value = english_data[key]
-                lang_code = parsed[fname].get("_language_code", fname)
+                inherited_sets[lang_idx].add(key)
-                print(f"  INFO: '{key}' missing in {lang_code}, using English fallback")
+                if verbose:
+                    print(
+                        f"  INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
+                    )
             row.append(value)
         translations[key] = row
 
@@ -195,10 +207,66 @@ def load_translations(
         extra = [k for k in data if not k.startswith("_") and k not in english_data]
         if extra:
             lang_code = data.get("_language_code", fname)
-            print(f"  WARNING: {lang_code} has keys not in English: {', '.join(extra)}")
+            if verbose:
+                print(
+                    f"  WARNING: {lang_code} has keys not in English: {', '.join(extra)}"
+                )
 
+    if verbose:
         print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys")
-    return language_codes, language_names, string_keys, translations
+    return language_codes, language_names, string_keys, translations, inherited_sets
+
+
+# ---------------------------------------------------------------------------
+# Unused-string detection
+# ---------------------------------------------------------------------------
+
+_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"}
+
+
+def find_used_string_keys(
+    src_dirs: List[str],
+    skip_filenames: Optional[Set[str]] = None,
+) -> Set[str]:
+    """
+    Scan C/C++ source files under *src_dirs* for STR_* identifiers.
+
+    Files whose basename appears in *skip_filenames* are skipped so that
+    the generated I18n files don't count as "usage" of themselves.
+
+    Returns the set of all STR_KEY names that appear at least once.
+    """
+    if skip_filenames is None:
+        skip_filenames = _GENERATED_FILENAMES
+
+    pattern = re.compile(r"\bSTR_[A-Z0-9_]+\b")
+    used: Set[str] = set()
+
+    for src_dir in src_dirs:
+        p = Path(src_dir)
+        if not p.is_dir():
+            continue
+        for f in p.rglob("*"):
+            if f.suffix not in {".cpp", ".h", ".c"}:
+                continue
+            if f.name in skip_filenames:
+                continue
+            try:
+                text = f.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            for m in pattern.finditer(text):
+                used.add(m.group(0))
+
+    return used
+
+
+def report_unused_keys(
+    string_keys: List[str],
+    used_keys: Set[str],
+) -> List[str]:
+    """Return a sorted list of keys from *string_keys* absent in *used_keys*."""
+    return [k for k in string_keys if k not in used_keys]
 
 
 # ---------------------------------------------------------------------------
@@ -207,23 +275,37 @@ def load_translations(
 
 LANG_ABBREVIATIONS = {
     "english": "EN",
-    "español": "ES", "espanol": "ES",
+    "español": "ES",
+    "espanol": "ES",
     "italiano": "IT",
     "svenska": "SV",
-    "français": "FR", "francais": "FR",
+    "français": "FR",
-    "deutsch": "DE", "german": "DE",
+    "francais": "FR",
+    "deutsch": "DE",
+    "german": "DE",
     "polski": "PL",
-    "português": "PT", "portugues": "PT", "português (brasil)": "PO",
+    "português": "PT",
-    "中文": "ZH", "chinese": "ZH",
+    "portugues": "PT",
-    "日本語": "JA", "japanese": "JA",
+    "português (brasil)": "PO",
-    "한국어": "KO", "korean": "KO",
+    "中文": "ZH",
-    "русский": "RU", "russian": "RU",
+    "chinese": "ZH",
-    "العربية": "AR", "arabic": "AR",
+    "日本語": "JA",
-    "עברית": "HE", "hebrew": "HE",
+    "japanese": "JA",
-    "فارسی": "FA", "persian": "FA",
+    "한국어": "KO",
+    "korean": "KO",
+    "русский": "RU",
+    "russian": "RU",
+    "العربية": "AR",
+    "arabic": "AR",
+    "עברית": "HE",
+    "hebrew": "HE",
+    "فارسی": "FA",
+    "persian": "FA",
     "čeština": "CS",
-    "türkçe": "TR", "turkish": "TR",
+    "türkçe": "TR",
-    "Қазақша": "KK", "kazakh": "KK",
+    "turkish": "TR",
+    "Қазақша": "KK",
+    "kazakh": "KK",
 }
 
 
@@ -267,7 +349,7 @@ def escape_cpp_string(s: str) -> List[str]:
 
         if ch == "\\" and i + 1 < len(s):
             nxt = s[i + 1]
-            if nxt in "ntr\"\\":
+            if nxt in 'ntr"\\':
                 current.append(ch + nxt)
                 i += 2
             elif nxt == "x" and i + 3 < len(s):
@@ -321,11 +403,11 @@ def format_cpp_string_literal(segments: List[str], indent: str = "    ") -> List
             last_space = -1
             idx = 0
             while idx <= MAX_CONTENT_LEN and idx < len(current):
-                if current[idx] == ' ':
+                if current[idx] == " ":
                     last_space = idx
 
                 # Handle escapes to step correctly
-                if current[idx] == '\\':
+                if current[idx] == "\\":
                     idx += 2
                 else:
                     idx += 1
@@ -340,7 +422,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = "    ") -> List
                 # No space, forced break at MAX_CONTENT_LEN (or slightly less)
                 cut_at = MAX_CONTENT_LEN
                 # Don't cut in the middle of an escape sequence
-                if current[cut_at - 1] == '\\':
+                if current[cut_at - 1] == "\\":
                     cut_at -= 1
 
                 lines.append(f'{indent}"{current[:cut_at]}"')
@@ -356,6 +438,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = "    ") -> List
 # Character-set computation
 # ---------------------------------------------------------------------------
 
+
 def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str:
     """Return a sorted string of every unique character used in a language."""
     chars = set()
@@ -369,11 +452,13 @@ def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -
 # Code generators
 # ---------------------------------------------------------------------------
 
+
 def generate_keys_header(
     languages: List[str],
     language_names: List[str],
     string_keys: List[str],
     output_path: str,
+    verbose: bool = False,
 ) -> None:
     """Generate I18nKeys.h."""
     lines: List[str] = [
@@ -382,13 +467,14 @@ def generate_keys_header(
         "",
         "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
         "",
-        "// Forward declaration for string arrays",
+        "// Forward declarations for flat string data blobs and offset tables",
         "namespace i18n_strings {",
     ]
 
     for code, name in zip(languages, language_names):
         abbrev = get_lang_abbreviation(code, name)
-        lines.append(f"extern const char* const STRINGS_{abbrev}[];")
+        lines.append(f"extern const char STRINGS_{abbrev}_DATA[];")
+        lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];")
 
     lines.append("}  // namespace i18n_strings")
     lines.append("")
@@ -420,17 +506,29 @@ def generate_keys_header(
     lines.append("};")
     lines.append("")
 
-    # getStringArray helper
+    # LangStrings struct
-    lines.append("// Helper function to get string array for a language")
+    lines.append("// Holds a flat string blob and its offset table for one language")
-    lines.append("inline const char* const* getStringArray(Language lang) {")
+    lines.append("struct LangStrings {")
+    lines.append("  const char* data;")
+    lines.append("  const uint16_t* offsets;")
+    lines.append("};")
+    lines.append("")
+
+    # getLanguageStrings helper
+    lines.append("// Helper function to get string data for a language")
+    lines.append("inline LangStrings getLanguageStrings(Language lang) {")
     lines.append("  switch (lang) {")
     for code, name in zip(languages, language_names):
         abbrev = get_lang_abbreviation(code, name)
         lines.append(f"    case Language::{code}:")
-        lines.append(f"      return i18n_strings::STRINGS_{abbrev};")
+        lines.append(
+            f"      return {{i18n_strings::STRINGS_{abbrev}_DATA, i18n_strings::OFFSETS_{abbrev}}};"
+        )
     first_abbrev = get_lang_abbreviation(languages[0], language_names[0])
     lines.append("    default:")
-    lines.append(f"      return i18n_strings::STRINGS_{first_abbrev};")
+    lines.append(
+        f"      return {{i18n_strings::STRINGS_{first_abbrev}_DATA, i18n_strings::OFFSETS_{first_abbrev}}};"
+    )
     lines.append("  }")
     lines.append("}")
     lines.append("")
@@ -463,22 +561,21 @@ def generate_keys_header(
     lines.append(
         "static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount(),"
     )
-    lines.append(
+    lines.append('              "SORTED_LANGUAGE_INDICES size mismatch");')
-        '              "SORTED_LANGUAGE_INDICES size mismatch");'
-    )
 
-    _write_file(output_path, lines)
+    _write_file(output_path, lines, verbose)
 
 
 def generate_strings_header(
     languages: List[str],
     language_names: List[str],
     output_path: str,
+    verbose: bool = False,
 ) -> None:
     """Generate I18nStrings.h."""
     lines: List[str] = [
         "#pragma once",
-        '#include <string>',
+        "#include <string>",
         "",
         '#include "I18nKeys.h"',
         "",
@@ -490,11 +587,12 @@ def generate_strings_header(
 
     for code, name in zip(languages, language_names):
         abbrev = get_lang_abbreviation(code, name)
-        lines.append(f"extern const char* const STRINGS_{abbrev}[];")
+        lines.append(f"extern const char STRINGS_{abbrev}_DATA[];")
+        lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];")
 
     lines.append("")
     lines.append("}  // namespace i18n_strings")
-    _write_file(output_path, lines)
+    _write_file(output_path, lines, verbose)
 
 
 def generate_strings_cpp(
@@ -503,6 +601,7 @@ def generate_strings_cpp(
     string_keys: List[str],
     translations: Dict[str, List[str]],
     output_path: str,
+    verbose: bool = False,
 ) -> None:
     """Generate I18nStrings.cpp."""
     lines: List[str] = [
@@ -529,18 +628,42 @@ def generate_strings_cpp(
     lines.append("};")
     lines.append("")
 
-    # Per-language string arrays
+    # Per-language flat string blobs and offset tables
     lines.append("namespace i18n_strings {")
     lines.append("")
 
     for lang_idx, (code, name) in enumerate(zip(languages, language_names)):
         abbrev = get_lang_abbreviation(code, name)
-        lines.append(f"const char* const STRINGS_{abbrev}[] = {{")
+        lang_strings = [translations[key][lang_idx] for key in string_keys]
 
-        for key in string_keys:
+        # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator)
-            text = translations[key][lang_idx]
+        offsets: List[int] = []
-            _append_string_entry(lines, text)
+        current_offset = 0
+        for s in lang_strings:
+            offsets.append(current_offset)
+            current_offset += len(s.encode("utf-8")) + 1
+        if current_offset > 65535:
+            raise ValueError(
+                f"Language {code}: total string data ({current_offset} bytes) "
+                "exceeds uint16_t offset range (65535)"
+            )
 
+        # Flat string data blob — all strings concatenated with \0 separators.
+        # clang-format off/on avoids reformatting of the adjacent string literals.
+        lines.append("// clang-format off")
+        lines.append(f"const char STRINGS_{abbrev}_DATA[] =")
+        for text in lang_strings:
+            _append_string_data_entry(lines, text)
+        lines.append(";")
+        lines.append("// clang-format on")
+        lines.append("")
+
+        # Offset table — one uint16_t per StrId
+        lines.append(f"const uint16_t OFFSETS_{abbrev}[] = {{")
+        chunk_size = 12
+        for i in range(0, len(offsets), chunk_size):
+            chunk = offsets[i : i + chunk_size]
+            lines.append("    " + ", ".join(str(o) for o in chunk) + ",")
         lines.append("};")
         lines.append("")
 
@@ -552,22 +675,84 @@ def generate_strings_cpp(
     for code, name in zip(languages, language_names):
         abbrev = get_lang_abbreviation(code, name)
         lines.append(
-            f"static_assert(sizeof(i18n_strings::STRINGS_{abbrev}) "
+            f"static_assert(sizeof(i18n_strings::OFFSETS_{abbrev}) "
-            f"/ sizeof(i18n_strings::STRINGS_{abbrev}[0]) =="
+            f"/ sizeof(i18n_strings::OFFSETS_{abbrev}[0]) =="
         )
         lines.append("                  static_cast<size_t>(StrId::_COUNT),")
-        lines.append(f'              "STRINGS_{abbrev} size mismatch");')
+        lines.append(f'              "OFFSETS_{abbrev} size mismatch");')
 
-    _write_file(output_path, lines)
+    _write_file(output_path, lines, verbose)
 
 
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
 
-def _append_string_entry(
+
-    lines: List[str], text: str, comment: str = ""
+def _print_language_table(
+    language_codes: List[str],
+    language_names: List[str],
+    inherited_sets: List[Set[str]],
+    string_keys: List[str],
+    unused_keys: Set[str],
 ) -> None:
+    """Print a per-language summary table."""
+    total = len(string_keys)
+    headers = ("Language", "Code", "Own", "Fallback", "Unused")
+
+    rows = []
+    for code, name, inherited in zip(language_codes, language_names, inherited_sets):
+        own = total - len(inherited)
+        fallback = len(inherited)
+        # strings this language translated but the code never calls
+        unused = len(unused_keys - inherited)
+        rows.append((name, code, str(own), str(fallback), str(unused)))
+
+    # EN first, then alphabetically by ISO code
+    rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
+
+    col_widths = [len(h) for h in headers]
+    for row in rows:
+        for i, cell in enumerate(row):
+            col_widths[i] = max(col_widths[i], len(cell))
+
+    fmt = "  ".join(f"{{:<{w}}}" for w in col_widths)
+    sep = "  ".join("-" * w for w in col_widths)
+
+    def _safe_print(line: str) -> None:
+        print(
+            line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode(
+                sys.stdout.encoding or "utf-8", errors="replace"
+            )
+        )
+
+    _safe_print(fmt.format(*headers))
+    _safe_print(sep)
+    for row in rows:
+        _safe_print(fmt.format(*row))
+    used = total - len(unused_keys)
+    print(
+        f"\n  Total: {total}  |  Used in code: {used}  |  Never used: {len(unused_keys)}"
+    )
+
+
+def _append_string_data_entry(lines: List[str], text: str) -> None:
+    """
+    Escape *text*, append a \\0 null separator, and format as indented C++
+    string literal lines for inclusion in a flat char data array blob.
+    """
+    segments = escape_cpp_string(text)
+    # Append the null entry separator to the last segment
+    if segments and segments[-1] != "":
+        segments[-1] += "\\0"
+    elif segments:
+        segments[-1] = "\\0"
+    else:
+        segments = ["\\0"]
+    lines.extend(format_cpp_string_literal(segments))
+
+
+def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None:
     """Escape *text*, format as indented C++ lines, append comma (and optional comment)."""
     segments = escape_cpp_string(text)
     formatted = format_cpp_string_literal(segments)
@@ -576,10 +761,11 @@ def _append_string_entry(
     lines.extend(formatted)
 
 
-def _write_file(path: str, lines: List[str]) -> None:
+def _write_file(path: str, lines: List[str], verbose: bool = False) -> None:
     with open(path, "w", encoding="utf-8", newline="\n") as f:
         f.write("\n".join(lines))
         f.write("\n")
+    if verbose:
         print(f"Generated: {path}")
 
 
@@ -587,10 +773,18 @@ def _write_file(path: str, lines: List[str]) -> None:
 # Main
 # ---------------------------------------------------------------------------
 
-def main(translations_dir=None, output_dir=None) -> None:
+
+def main(
+    translations_dir: Optional[str] = None,
+    output_dir: Optional[str] = None,
+    src_dirs: Optional[List[str]] = None,
+    strip_unused: bool = False,
+    verbose: bool = False,
+) -> None:
     # Default paths (relative to project root)
     default_translations_dir = "lib/I18n/translations"
     default_output_dir = "lib/I18n/"
+    default_src_dirs = ["src", "lib"]
 
     if translations_dir is None or output_dir is None:
         if len(sys.argv) == 3:
@@ -601,6 +795,8 @@ def main(translations_dir=None, output_dir=None) -> None:
             translations_dir = default_translations_dir
             output_dir = default_output_dir
 
+    if src_dirs is None:
+        src_dirs = default_src_dirs
 
     if not os.path.isdir(translations_dir):
         print(f"Error: Translations directory not found: {translations_dir}")
@@ -610,26 +806,68 @@ def main(translations_dir=None, output_dir=None) -> None:
         print(f"Error: Output directory not found: {output_dir}")
         sys.exit(1)
 
+    if verbose:
         print(f"Reading translations from: {translations_dir}")
         print(f"Output directory: {output_dir}")
         print()
 
     try:
-        languages, language_names, string_keys, translations = load_translations(
+        languages, language_names, string_keys, translations, inherited_sets = (
-            translations_dir
+            load_translations(translations_dir, verbose)
         )
 
+        # --- Unused-string detection ---
+        scan_dirs = [d for d in src_dirs if os.path.isdir(d)]
+        if scan_dirs:
+            used_keys = find_used_string_keys(scan_dirs)
+            unused_set = set(report_unused_keys(string_keys, used_keys))
+        else:
+            used_keys = set(string_keys)
+            unused_set = set()
+
+        _print_language_table(
+            languages, language_names, inherited_sets, string_keys, unused_set
+        )
+        print()
+
+        if verbose and unused_set:
+            print(f"  Unused keys ({len(unused_set)}):")
+            for key in sorted(unused_set):
+                print(f"    - {key}")
+            print()
+
+        if unused_set and strip_unused:
+            string_keys = [k for k in string_keys if k not in unused_set]
+            translations = {
+                k: v for k, v in translations.items() if k not in unused_set
+            }
+            inherited_sets = [s - unused_set for s in inherited_sets]
+            print(f"  Stripping {len(unused_set)} unused string(s) from output.")
+
         out = Path(output_dir)
-        generate_keys_header(languages, language_names, string_keys, str(out / "I18nKeys.h"))
+        generate_keys_header(
-        generate_strings_header(languages, language_names, str(out / "I18nStrings.h"))
+            languages, language_names, string_keys, str(out / "I18nKeys.h"), verbose
+        )
+        generate_strings_header(
+            languages, language_names, str(out / "I18nStrings.h"), verbose
+        )
         generate_strings_cpp(
-            languages, language_names, string_keys, translations, str(out / "I18nStrings.cpp")
+            languages,
+            language_names,
+            string_keys,
+            translations,
+            str(out / "I18nStrings.cpp"),
+            verbose,
         )
 
         print()
-        print("✓ Code generation complete!")
+        print("Code generation complete!")
         print(f"  Languages: {len(languages)}")
         print(f"  String keys: {len(string_keys)}")
+        if unused_set and not strip_unused:
+            print(
+                f"  Unused keys: {len(unused_set)} (pass --strip-unused to remove them)"
+            )
 
     except Exception as e:
         print(f"\nError: {e}")
@@ -637,11 +875,52 @@ def main(translations_dir=None, output_dir=None) -> None:
 
 
 if __name__ == "__main__":
-    main()
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Generate I18n C++ files from per-language YAML translations."
+    )
+    parser.add_argument(
+        "translations_dir",
+        nargs="?",
+        default=None,
+        help="Path to the translations directory (default: lib/I18n/translations)",
+    )
+    parser.add_argument(
+        "output_dir",
+        nargs="?",
+        default=None,
+        help="Path to the output directory (default: lib/I18n/)",
+    )
+    parser.add_argument(
+        "--src-dirs",
+        nargs="+",
+        metavar="DIR",
+        default=None,
+        help="Source directories to scan for STR_* usage (default: src lib)",
+    )
+    parser.add_argument(
+        "--strip-unused",
+        action="store_true",
+        help="Remove unused STR_* keys from the generated output",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Print per-key INFO/WARNING messages and file generation details",
+    )
+    args = parser.parse_args()
+    main(
+        args.translations_dir,
+        args.output_dir,
+        args.src_dirs,
+        args.strip_unused,
+        args.verbose,
+    )
 else:
     try:
         Import("env")
-        print("Running i18n generation script from PlatformIO...")
         main()
     except NameError:
         pass