From 960cb91b23abe0651bb221d1065a89403639639f Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 15:10:41 +0100 Subject: [PATCH 1/8] Optimize output --- lib/I18n/I18n.cpp | 4 +- scripts/gen_i18n.py | 423 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 353 insertions(+), 74 deletions(-) diff --git a/lib/I18n/I18n.cpp b/lib/I18n/I18n.cpp index 55cf8fc4..c77fce9a 100644 --- a/lib/I18n/I18n.cpp +++ b/lib/I18n/I18n.cpp @@ -24,8 +24,8 @@ const char* I18n::get(StrId id) const { } // Use generated helper function - no hardcoded switch needed! - const char* const* strings = getStringArray(_language); - return strings[index]; + const LangStrings lang = getLanguageStrings(_language); + return lang.data + lang.offsets[index]; } void I18n::setLanguage(Language lang) { diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 74d4b197..5d553427 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -15,24 +15,32 @@ Each YAML file must contain: The English file is the reference. Missing keys in other languages are automatically filled from English, with a warning. -Usage: - python gen_i18n.py +By default the script scans the src/ and lib/ trees for STR_* references and +reports any translation keys that are never used. Pass --strip-unused to +omit those keys from the generated output entirely. -Example: +Usage: + python gen_i18n.py [translations_dir [output_dir]] [options] + +Examples: + python gen_i18n.py python gen_i18n.py lib/I18n/translations lib/I18n/ + python gen_i18n.py --strip-unused + python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont """ import sys import os import re from pathlib import Path -from typing import List, Dict, Tuple +from typing import Dict, List, Optional, Set, Tuple # --------------------------------------------------------------------------- # YAML file reading (simple key: "value" format, no PyYAML dependency) # --------------------------------------------------------------------------- + def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str: """ Process escape sequences in a YAML value string. @@ -51,9 +59,7 @@ def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str elif nxt == "n": result.append("\n") else: - raise ValueError( - f"{filepath}:{line_num}: unknown escape '\\{nxt}'" - ) + raise ValueError(f"{filepath}:{line_num}: unknown escape '\\{nxt}'") i += 2 else: result.append(raw[i]) @@ -103,9 +109,11 @@ def parse_yaml_file(filepath: str) -> Dict[str, str]: # Load all languages from a directory of YAML files # --------------------------------------------------------------------------- + def load_translations( translations_dir: str, -) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]]]: + verbose: bool = False, +) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]: """ Read every YAML file in *translations_dir* and return: language_codes e.g. ["EN", "ES", ...] @@ -174,16 +182,20 @@ def load_translations( raise ValueError(f"Invalid C++ identifier in English file: '{key}'") # Build translations dict, filling missing keys from English + inherited_sets: List[Set[str]] = [set() for _ in ordered_files] translations: Dict[str, List[str]] = {} for key in string_keys: row: List[str] = [] - for fname in ordered_files: + for lang_idx, fname in enumerate(ordered_files): data = parsed[fname] value = data.get(key, "") if not value.strip() and fname != english_file: value = english_data[key] - lang_code = parsed[fname].get("_language_code", fname) - print(f" INFO: '{key}' missing in {lang_code}, using English fallback") + inherited_sets[lang_idx].add(key) + if verbose: + print( + f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback" + ) row.append(value) translations[key] = row @@ -195,10 +207,66 @@ def load_translations( extra = [k for k in data if not k.startswith("_") and k not in english_data] if extra: lang_code = data.get("_language_code", fname) - print(f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}") + if verbose: + print( + f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}" + ) - print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") - return language_codes, language_names, string_keys, translations + if verbose: + print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") + return language_codes, language_names, string_keys, translations, inherited_sets + + +# --------------------------------------------------------------------------- +# Unused-string detection +# --------------------------------------------------------------------------- + +_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"} + + +def find_used_string_keys( + src_dirs: List[str], + skip_filenames: Optional[Set[str]] = None, +) -> Set[str]: + """ + Scan C/C++ source files under *src_dirs* for STR_* identifiers. + + Files whose basename appears in *skip_filenames* are skipped so that + the generated I18n files don't count as "usage" of themselves. + + Returns the set of all STR_KEY names that appear at least once. + """ + if skip_filenames is None: + skip_filenames = _GENERATED_FILENAMES + + pattern = re.compile(r"\bSTR_[A-Z0-9_]+\b") + used: Set[str] = set() + + for src_dir in src_dirs: + p = Path(src_dir) + if not p.is_dir(): + continue + for f in p.rglob("*"): + if f.suffix not in {".cpp", ".h", ".c"}: + continue + if f.name in skip_filenames: + continue + try: + text = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for m in pattern.finditer(text): + used.add(m.group(0)) + + return used + + +def report_unused_keys( + string_keys: List[str], + used_keys: Set[str], +) -> List[str]: + """Return a sorted list of keys from *string_keys* absent in *used_keys*.""" + return [k for k in string_keys if k not in used_keys] # --------------------------------------------------------------------------- @@ -207,23 +275,37 @@ def load_translations( LANG_ABBREVIATIONS = { "english": "EN", - "español": "ES", "espanol": "ES", + "español": "ES", + "espanol": "ES", "italiano": "IT", "svenska": "SV", - "français": "FR", "francais": "FR", - "deutsch": "DE", "german": "DE", + "français": "FR", + "francais": "FR", + "deutsch": "DE", + "german": "DE", "polski": "PL", - "português": "PT", "portugues": "PT", "português (brasil)": "PO", - "中文": "ZH", "chinese": "ZH", - "日本語": "JA", "japanese": "JA", - "한국어": "KO", "korean": "KO", - "русский": "RU", "russian": "RU", - "العربية": "AR", "arabic": "AR", - "עברית": "HE", "hebrew": "HE", - "فارسی": "FA", "persian": "FA", + "português": "PT", + "portugues": "PT", + "português (brasil)": "PO", + "中文": "ZH", + "chinese": "ZH", + "日本語": "JA", + "japanese": "JA", + "한국어": "KO", + "korean": "KO", + "русский": "RU", + "russian": "RU", + "العربية": "AR", + "arabic": "AR", + "עברית": "HE", + "hebrew": "HE", + "فارسی": "FA", + "persian": "FA", "čeština": "CS", - "türkçe": "TR", "turkish": "TR", - "Қазақша": "KK", "kazakh": "KK", + "türkçe": "TR", + "turkish": "TR", + "Қазақша": "KK", + "kazakh": "KK", } @@ -267,12 +349,12 @@ def escape_cpp_string(s: str) -> List[str]: if ch == "\\" and i + 1 < len(s): nxt = s[i + 1] - if nxt in "ntr\"\\": + if nxt in 'ntr"\\': current.append(ch + nxt) i += 2 elif nxt == "x" and i + 3 < len(s): current.append(s[i : i + 4]) - _flush() # segment break after hex + _flush() # segment break after hex i += 4 else: current.append("\\\\") @@ -286,7 +368,7 @@ def escape_cpp_string(s: str) -> List[str]: else: for byte in ch.encode("utf-8"): current.append(f"\\x{byte:02X}") - _flush() # segment break after hex + _flush() # segment break after hex i += 1 # Flush remaining content @@ -321,11 +403,11 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List last_space = -1 idx = 0 while idx <= MAX_CONTENT_LEN and idx < len(current): - if current[idx] == ' ': + if current[idx] == " ": last_space = idx # Handle escapes to step correctly - if current[idx] == '\\': + if current[idx] == "\\": idx += 2 else: idx += 1 @@ -340,7 +422,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List # No space, forced break at MAX_CONTENT_LEN (or slightly less) cut_at = MAX_CONTENT_LEN # Don't cut in the middle of an escape sequence - if current[cut_at - 1] == '\\': + if current[cut_at - 1] == "\\": cut_at -= 1 lines.append(f'{indent}"{current[:cut_at]}"') @@ -356,6 +438,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List # Character-set computation # --------------------------------------------------------------------------- + def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str: """Return a sorted string of every unique character used in a language.""" chars = set() @@ -369,11 +452,13 @@ def compute_character_set(translations: Dict[str, List[str]], lang_index: int) - # Code generators # --------------------------------------------------------------------------- + def generate_keys_header( languages: List[str], language_names: List[str], string_keys: List[str], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nKeys.h.""" lines: List[str] = [ @@ -382,13 +467,14 @@ def generate_keys_header( "", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", "", - "// Forward declaration for string arrays", + "// Forward declarations for flat string data blobs and offset tables", "namespace i18n_strings {", ] for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) - lines.append(f"extern const char* const STRINGS_{abbrev}[];") + lines.append(f"extern const char STRINGS_{abbrev}_DATA[];") + lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];") lines.append("} // namespace i18n_strings") lines.append("") @@ -420,17 +506,29 @@ def generate_keys_header( lines.append("};") lines.append("") - # getStringArray helper - lines.append("// Helper function to get string array for a language") - lines.append("inline const char* const* getStringArray(Language lang) {") + # LangStrings struct + lines.append("// Holds a flat string blob and its offset table for one language") + lines.append("struct LangStrings {") + lines.append(" const char* data;") + lines.append(" const uint16_t* offsets;") + lines.append("};") + lines.append("") + + # getLanguageStrings helper + lines.append("// Helper function to get string data for a language") + lines.append("inline LangStrings getLanguageStrings(Language lang) {") lines.append(" switch (lang) {") for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) lines.append(f" case Language::{code}:") - lines.append(f" return i18n_strings::STRINGS_{abbrev};") + lines.append( + f" return {{i18n_strings::STRINGS_{abbrev}_DATA, i18n_strings::OFFSETS_{abbrev}}};" + ) first_abbrev = get_lang_abbreviation(languages[0], language_names[0]) lines.append(" default:") - lines.append(f" return i18n_strings::STRINGS_{first_abbrev};") + lines.append( + f" return {{i18n_strings::STRINGS_{first_abbrev}_DATA, i18n_strings::OFFSETS_{first_abbrev}}};" + ) lines.append(" }") lines.append("}") lines.append("") @@ -463,22 +561,21 @@ def generate_keys_header( lines.append( "static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount()," ) - lines.append( - ' "SORTED_LANGUAGE_INDICES size mismatch");' - ) + lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");') - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) def generate_strings_header( languages: List[str], language_names: List[str], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nStrings.h.""" lines: List[str] = [ "#pragma once", - '#include ', + "#include ", "", '#include "I18nKeys.h"', "", @@ -490,11 +587,12 @@ def generate_strings_header( for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) - lines.append(f"extern const char* const STRINGS_{abbrev}[];") + lines.append(f"extern const char STRINGS_{abbrev}_DATA[];") + lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];") lines.append("") lines.append("} // namespace i18n_strings") - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) def generate_strings_cpp( @@ -503,6 +601,7 @@ def generate_strings_cpp( string_keys: List[str], translations: Dict[str, List[str]], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nStrings.cpp.""" lines: List[str] = [ @@ -529,18 +628,42 @@ def generate_strings_cpp( lines.append("};") lines.append("") - # Per-language string arrays + # Per-language flat string blobs and offset tables lines.append("namespace i18n_strings {") lines.append("") for lang_idx, (code, name) in enumerate(zip(languages, language_names)): abbrev = get_lang_abbreviation(code, name) - lines.append(f"const char* const STRINGS_{abbrev}[] = {{") + lang_strings = [translations[key][lang_idx] for key in string_keys] - for key in string_keys: - text = translations[key][lang_idx] - _append_string_entry(lines, text) + # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator) + offsets: List[int] = [] + current_offset = 0 + for s in lang_strings: + offsets.append(current_offset) + current_offset += len(s.encode("utf-8")) + 1 + if current_offset > 65535: + raise ValueError( + f"Language {code}: total string data ({current_offset} bytes) " + "exceeds uint16_t offset range (65535)" + ) + # Flat string data blob — all strings concatenated with \0 separators. + # clang-format off/on avoids reformatting of the adjacent string literals. + lines.append("// clang-format off") + lines.append(f"const char STRINGS_{abbrev}_DATA[] =") + for text in lang_strings: + _append_string_data_entry(lines, text) + lines.append(";") + lines.append("// clang-format on") + lines.append("") + + # Offset table — one uint16_t per StrId + lines.append(f"const uint16_t OFFSETS_{abbrev}[] = {{") + chunk_size = 12 + for i in range(0, len(offsets), chunk_size): + chunk = offsets[i : i + chunk_size] + lines.append(" " + ", ".join(str(o) for o in chunk) + ",") lines.append("};") lines.append("") @@ -552,22 +675,84 @@ def generate_strings_cpp( for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) lines.append( - f"static_assert(sizeof(i18n_strings::STRINGS_{abbrev}) " - f"/ sizeof(i18n_strings::STRINGS_{abbrev}[0]) ==" + f"static_assert(sizeof(i18n_strings::OFFSETS_{abbrev}) " + f"/ sizeof(i18n_strings::OFFSETS_{abbrev}[0]) ==" ) lines.append(" static_cast(StrId::_COUNT),") - lines.append(f' "STRINGS_{abbrev} size mismatch");') + lines.append(f' "OFFSETS_{abbrev} size mismatch");') - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _append_string_entry( - lines: List[str], text: str, comment: str = "" + +def _print_language_table( + language_codes: List[str], + language_names: List[str], + inherited_sets: List[Set[str]], + string_keys: List[str], + unused_keys: Set[str], ) -> None: + """Print a per-language summary table.""" + total = len(string_keys) + headers = ("Language", "Code", "Own", "Fallback", "Unused") + + rows = [] + for code, name, inherited in zip(language_codes, language_names, inherited_sets): + own = total - len(inherited) + fallback = len(inherited) + # strings this language translated but the code never calls + unused = len(unused_keys - inherited) + rows.append((name, code, str(own), str(fallback), str(unused))) + + # EN first, then alphabetically by ISO code + rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1])) + + col_widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + col_widths[i] = max(col_widths[i], len(cell)) + + fmt = " ".join(f"{{:<{w}}}" for w in col_widths) + sep = " ".join("-" * w for w in col_widths) + + def _safe_print(line: str) -> None: + print( + line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode( + sys.stdout.encoding or "utf-8", errors="replace" + ) + ) + + _safe_print(fmt.format(*headers)) + _safe_print(sep) + for row in rows: + _safe_print(fmt.format(*row)) + used = total - len(unused_keys) + print( + f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}" + ) + + +def _append_string_data_entry(lines: List[str], text: str) -> None: + """ + Escape *text*, append a \\0 null separator, and format as indented C++ + string literal lines for inclusion in a flat char data array blob. + """ + segments = escape_cpp_string(text) + # Append the null entry separator to the last segment + if segments and segments[-1] != "": + segments[-1] += "\\0" + elif segments: + segments[-1] = "\\0" + else: + segments = ["\\0"] + lines.extend(format_cpp_string_literal(segments)) + + +def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None: """Escape *text*, format as indented C++ lines, append comma (and optional comment).""" segments = escape_cpp_string(text) formatted = format_cpp_string_literal(segments) @@ -576,21 +761,30 @@ def _append_string_entry( lines.extend(formatted) -def _write_file(path: str, lines: List[str]) -> None: +def _write_file(path: str, lines: List[str], verbose: bool = False) -> None: with open(path, "w", encoding="utf-8", newline="\n") as f: f.write("\n".join(lines)) f.write("\n") - print(f"Generated: {path}") + if verbose: + print(f"Generated: {path}") # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -def main(translations_dir=None, output_dir=None) -> None: + +def main( + translations_dir: Optional[str] = None, + output_dir: Optional[str] = None, + src_dirs: Optional[List[str]] = None, + strip_unused: bool = False, + verbose: bool = False, +) -> None: # Default paths (relative to project root) default_translations_dir = "lib/I18n/translations" default_output_dir = "lib/I18n/" + default_src_dirs = ["src", "lib"] if translations_dir is None or output_dir is None: if len(sys.argv) == 3: @@ -601,6 +795,8 @@ def main(translations_dir=None, output_dir=None) -> None: translations_dir = default_translations_dir output_dir = default_output_dir + if src_dirs is None: + src_dirs = default_src_dirs if not os.path.isdir(translations_dir): print(f"Error: Translations directory not found: {translations_dir}") @@ -610,26 +806,68 @@ def main(translations_dir=None, output_dir=None) -> None: print(f"Error: Output directory not found: {output_dir}") sys.exit(1) - print(f"Reading translations from: {translations_dir}") - print(f"Output directory: {output_dir}") - print() + if verbose: + print(f"Reading translations from: {translations_dir}") + print(f"Output directory: {output_dir}") + print() try: - languages, language_names, string_keys, translations = load_translations( - translations_dir + languages, language_names, string_keys, translations, inherited_sets = ( + load_translations(translations_dir, verbose) ) + # --- Unused-string detection --- + scan_dirs = [d for d in src_dirs if os.path.isdir(d)] + if scan_dirs: + used_keys = find_used_string_keys(scan_dirs) + unused_set = set(report_unused_keys(string_keys, used_keys)) + else: + used_keys = set(string_keys) + unused_set = set() + + _print_language_table( + languages, language_names, inherited_sets, string_keys, unused_set + ) + print() + + if verbose and unused_set: + print(f" Unused keys ({len(unused_set)}):") + for key in sorted(unused_set): + print(f" - {key}") + print() + + if unused_set and strip_unused: + string_keys = [k for k in string_keys if k not in unused_set] + translations = { + k: v for k, v in translations.items() if k not in unused_set + } + inherited_sets = [s - unused_set for s in inherited_sets] + print(f" Stripping {len(unused_set)} unused string(s) from output.") + out = Path(output_dir) - generate_keys_header(languages, language_names, string_keys, str(out / "I18nKeys.h")) - generate_strings_header(languages, language_names, str(out / "I18nStrings.h")) + generate_keys_header( + languages, language_names, string_keys, str(out / "I18nKeys.h"), verbose + ) + generate_strings_header( + languages, language_names, str(out / "I18nStrings.h"), verbose + ) generate_strings_cpp( - languages, language_names, string_keys, translations, str(out / "I18nStrings.cpp") + languages, + language_names, + string_keys, + translations, + str(out / "I18nStrings.cpp"), + verbose, ) print() - print("✓ Code generation complete!") + print("Code generation complete!") print(f" Languages: {len(languages)}") print(f" String keys: {len(string_keys)}") + if unused_set and not strip_unused: + print( + f" Unused keys: {len(unused_set)} (pass --strip-unused to remove them)" + ) except Exception as e: print(f"\nError: {e}") @@ -637,11 +875,52 @@ def main(translations_dir=None, output_dir=None) -> None: if __name__ == "__main__": - main() + import argparse + + parser = argparse.ArgumentParser( + description="Generate I18n C++ files from per-language YAML translations." + ) + parser.add_argument( + "translations_dir", + nargs="?", + default=None, + help="Path to the translations directory (default: lib/I18n/translations)", + ) + parser.add_argument( + "output_dir", + nargs="?", + default=None, + help="Path to the output directory (default: lib/I18n/)", + ) + parser.add_argument( + "--src-dirs", + nargs="+", + metavar="DIR", + default=None, + help="Source directories to scan for STR_* usage (default: src lib)", + ) + parser.add_argument( + "--strip-unused", + action="store_true", + help="Remove unused STR_* keys from the generated output", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print per-key INFO/WARNING messages and file generation details", + ) + args = parser.parse_args() + main( + args.translations_dir, + args.output_dir, + args.src_dirs, + args.strip_unused, + args.verbose, + ) else: try: Import("env") - print("Running i18n generation script from PlatformIO...") main() except NameError: pass From 9b6035820da57e554e7f47a339d9f1b309d91353 Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 15:31:59 +0100 Subject: [PATCH 2/8] Show sizes, too --- scripts/gen_i18n.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 5d553427..89c77473 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -695,18 +695,21 @@ def _print_language_table( inherited_sets: List[Set[str]], string_keys: List[str], unused_keys: Set[str], + data_sizes: List[int], ) -> None: """Print a per-language summary table.""" total = len(string_keys) - headers = ("Language", "Code", "Own", "Fallback", "Unused") + headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)") rows = [] - for code, name, inherited in zip(language_codes, language_names, inherited_sets): + for code, name, inherited, size in zip( + language_codes, language_names, inherited_sets, data_sizes + ): own = total - len(inherited) fallback = len(inherited) # strings this language translated but the code never calls unused = len(unused_keys - inherited) - rows.append((name, code, str(own), str(fallback), str(unused))) + rows.append((name, code, str(own), str(fallback), str(unused), str(size))) # EN first, then alphabetically by ISO code rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1])) @@ -731,9 +734,15 @@ def _print_language_table( for row in rows: _safe_print(fmt.format(*row)) used = total - len(unused_keys) + total_size = sum(data_sizes) + offset_table_size = len(rows) * len(string_keys) * 2 print( f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}" ) + print( + f" Flash: {total_size:,} B string data + {offset_table_size:,} B offset tables" + f" = {total_size + offset_table_size:,} B total" + ) def _append_string_data_entry(lines: List[str], text: str) -> None: @@ -825,8 +834,20 @@ def main( used_keys = set(string_keys) unused_set = set() + # Compute per-language data blob sizes: + # sum of UTF-8 byte length + 1 (null terminator) per string + data_sizes = [ + sum(len(translations[k][i].encode("utf-8")) + 1 for k in string_keys) + for i in range(len(languages)) + ] + _print_language_table( - languages, language_names, inherited_sets, string_keys, unused_set + languages, + language_names, + inherited_sets, + string_keys, + unused_set, + data_sizes, ) print() From 039337b4eb365499979afb4c5f7c6cf7b6adcb0c Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 15:41:25 +0100 Subject: [PATCH 3/8] Show sizes --- scripts/gen_i18n.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 89c77473..c7d73099 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -735,14 +735,27 @@ def _print_language_table( _safe_print(fmt.format(*row)) used = total - len(unused_keys) total_size = sum(data_sizes) - offset_table_size = len(rows) * len(string_keys) * 2 + n_lang = len(rows) + n_keys = len(string_keys) + # Current layout: uint16_t offset table (2 B per string per language) + offset_table_size = n_lang * n_keys * 2 + current_total = total_size + offset_table_size + # Previous layout: const char* pointer array (4 B per string per language) + old_pointer_table_size = n_lang * n_keys * 4 + old_total = total_size + old_pointer_table_size + saved = old_total - current_total print( f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}" ) print( - f" Flash: {total_size:,} B string data + {offset_table_size:,} B offset tables" - f" = {total_size + offset_table_size:,} B total" + f" Flash (now): {total_size:>7,} B strings + {offset_table_size:>6,} B offset tables (uint16_t)" + f" = {current_total:>7,} B" ) + print( + f" Flash (before): {total_size:>7,} B strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)" + f" = {old_total:>7,} B" + ) + print(f" Saved by offset tables: {saved:,} B") def _append_string_data_entry(lines: List[str], text: str) -> None: From 72a0d2a6119f6e2301ce53482496f01c69f488f1 Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 15:44:38 +0100 Subject: [PATCH 4/8] chore: remove unused STR_CALIBRE from German translations --- lib/I18n/translations/german.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/I18n/translations/german.yaml b/lib/I18n/translations/german.yaml index aebff633..f1712bd7 100644 --- a/lib/I18n/translations/german.yaml +++ b/lib/I18n/translations/german.yaml @@ -118,7 +118,6 @@ STR_CHECK_UPDATES: "Nach Updates suchen" STR_LANGUAGE: "Sprache" STR_SELECT_WALLPAPER: "Bildauswahl Standby" STR_CLEAR_READING_CACHE: "Lese-Cache leeren" -STR_CALIBRE: "Calibre" STR_USERNAME: "Benutzername" STR_PASSWORD: "Passwort" STR_SYNC_SERVER_URL: "Sync-Server-URL" From 2e8ee680a72951461117fb70628b03142fb2c464 Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 15:47:13 +0100 Subject: [PATCH 5/8] perf: strip unused strings during PlatformIO build --- scripts/gen_i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index c7d73099..ff9e5d79 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -955,6 +955,6 @@ if __name__ == "__main__": else: try: Import("env") - main() + main(strip_unused=True) except NameError: pass From 7c0095e5c5289cb7b160c8755412d92b1f38533f Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 20:03:46 +0100 Subject: [PATCH 6/8] Remove no longer needed include --- scripts/gen_i18n.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index ff9e5d79..1d8d3e48 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -575,8 +575,6 @@ def generate_strings_header( """Generate I18nStrings.h.""" lines: List[str] = [ "#pragma once", - "#include ", - "", '#include "I18nKeys.h"', "", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", From 2419cc43bf2f6f1c24bd4a9c146c874f59f40e41 Mon Sep 17 00:00:00 2001 From: jpirnay Date: Mon, 16 Mar 2026 23:52:13 +0100 Subject: [PATCH 7/8] fix missing header --- scripts/gen_i18n.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 1d8d3e48..89a0ba8f 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -604,6 +604,7 @@ def generate_strings_cpp( """Generate I18nStrings.cpp.""" lines: List[str] = [ '#include "I18nStrings.h"', + "#include ", "", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", "", From f23d01093ebb6f4a1d6301c9ebc0eaccc6b66e43 Mon Sep 17 00:00:00 2001 From: jpirnay Date: Tue, 17 Mar 2026 07:07:47 +0100 Subject: [PATCH 8/8] Amend regex pattern for tr identification --- scripts/gen_i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 89a0ba8f..55aef15b 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -239,7 +239,7 @@ def find_used_string_keys( if skip_filenames is None: skip_filenames = _GENERATED_FILENAMES - pattern = re.compile(r"\bSTR_[A-Z0-9_]+\b") + pattern = re.compile(r"\bSTR_[A-Za-z0-9_]+\b") used: Set[str] = set() for src_dir in src_dirs: