From d53c8b0e0ede349ad972262525e0fa42ee75ea9c Mon Sep 17 00:00:00 2001 From: jpirnay Date: Thu, 30 Apr 2026 20:57:19 +0200 Subject: [PATCH] perf: replace i18n pointer tables with offset tables, strip unused strings (#1408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary * **What is the goal of this PR?** Reduce the flash footprint of the i18n string data and tooling improvements to `gen_i18n.py`. * **What changes are included?** ### 1. `lib/I18n/I18n.cpp` — use offset-based lookup `I18n::get()` previously dereferenced a `const char* const*` pointer array. It now uses a two-field `LangStrings` struct (a flat char blob + a `uint16_t` offset table) generated for each language: ```cpp // before const char* const* strings = getStringArray(_language); return strings[index]; // after const LangStrings lang = getLanguageStrings(_language); return lang.data + lang.offsets[index]; ``` Lookup cost is unchanged — still O(1), one array load and one addition. ### 2. `scripts/gen_i18n.py` — new generated layout Each language's string data is now emitted as: - **`STRINGS_XX_DATA[]`** — a single `const char[]` blob of all strings concatenated with `\0` separators. - **`OFFSETS_XX[]`** — a `uint16_t` array of one byte-offset per `StrId` into the blob. Previously each language had a `const char* const STRINGS_XX[]` pointer array (4 bytes/entry on ESP32-C3). #### Flash savings | Table type | Size per language | 19 languages | |---|---|---| | `const char*` pointer array (before) | `339 × 4 = 1,356 B` | **25,764 B** | | `uint16_t` offset table (after) | `339 × 2 = 678 B` | **12,882 B** | | **Saved** | | **12,882 B (~12.6 KB)** | String data size is unchanged — 133,092 B across 19 languages. **Total: 158,856 B → 145,974 B** (pointer tables → offset tables). ### 3. Build-time stripping of unused strings `gen_i18n.py` now scans the `src/` and `lib/` trees for `STR_*` references and, during a PlatformIO build, automatically omits the 52 strings that are defined in YAML but never referenced in code. This further reduces the compiled output from 339 → 287 string keys per language. --- ## `gen_i18n.py` CLI reference ``` python gen_i18n.py [translations_dir [output_dir]] [options] ``` | Argument / Flag | Default | Description | |---|---|---| | `translations_dir` | `lib/I18n/translations` | Path to the per-language YAML files | | `output_dir` | `lib/I18n/` | Where to write the generated `.h` / `.cpp` files | | `--src-dirs DIR [DIR …]` | `src lib` | Directories scanned for `STR_*` usage | | `--strip-unused` | off | Remove unreferenced `STR_*` keys from generated output | | `--verbose` / `-v` | off | Print per-key INFO/WARNING messages and `Generated:` lines | The PlatformIO build (SCons `pre:` hook) calls `main(strip_unused=True)` automatically, so unused strings are always stripped from firmware builds without any manual flag. **Default run output** (no flags) shows a per-language summary table: ``` Language Code Own Fallback Unused Data (B) ------------------ ---- --- -------- ------ -------- English EN 339 0 52 5,266 Belarusian BE 310 29 52 9,673 … Total: 339 | Used in code: 287 | Never used: 52 Flash (now): 133,092 B strings + 12,882 B offset tables (uint16_t) = 145,974 B Flash (before): 133,092 B strings + 25,764 B pointer tables (ptr32) = 158,856 B Saved by offset tables: 12,882 B ``` --- ### AI Usage Did you use AI tools to help write this code? **YES** (GitHub Copilot) --------- Co-authored-by: Zach Nelson --- lib/I18n/I18n.cpp | 67 +++++- lib/I18n/I18n.h | 1 + scripts/gen_i18n.py | 520 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 472 insertions(+), 116 deletions(-) diff --git a/lib/I18n/I18n.cpp b/lib/I18n/I18n.cpp index 545e1436..9ccb3cd4 100644 --- a/lib/I18n/I18n.cpp +++ b/lib/I18n/I18n.cpp @@ -4,13 +4,15 @@ #include #include +#include + #include "I18nStrings.h" using namespace i18n_strings; // Settings file path static constexpr const char* SETTINGS_FILE = "/.crosspoint/language.bin"; -static constexpr uint8_t SETTINGS_VERSION = 1; +static constexpr uint8_t SETTINGS_VERSION = 2; I18n& I18n::getInstance() { static I18n instance; @@ -24,8 +26,8 @@ const char* I18n::get(StrId id) const { } // Use generated helper function - no hardcoded switch needed! - const char* const* strings = getStringArray(_language); - return strings[index]; + const LangStrings lang = getLanguageStrings(_language); + return lang.data + lang.offsets[index]; } void I18n::setLanguage(Language lang) { @@ -44,6 +46,14 @@ const char* I18n::getLanguageName(Language lang) const { return LANGUAGE_NAMES[index]; } +const char* I18n::getLanguageCode(Language lang) const { + const auto index = static_cast(lang); + if (index >= static_cast(Language::_COUNT)) { + return LANGUAGE_CODES[0]; + } + return LANGUAGE_CODES[index]; +} + void I18n::saveSettings() { Storage.mkdir("/.crosspoint"); @@ -54,10 +64,11 @@ void I18n::saveSettings() { } serialization::writePod(file, SETTINGS_VERSION); - serialization::writePod(file, static_cast(_language)); + serialization::writeString(file, getLanguageCode(_language)); file.close(); - Serial.printf("[I18N] Settings saved: language=%d\n", static_cast(_language)); + Serial.printf("[I18N] Settings saved: language=%d code=%s\n", static_cast(_language), + getLanguageCode(_language)); } void I18n::loadSettings() { @@ -69,17 +80,49 @@ void I18n::loadSettings() { uint8_t version; serialization::readPod(file, version); - if (version != SETTINGS_VERSION) { - Serial.printf("[I18N] Settings version mismatch\n"); + + if (version == SETTINGS_VERSION) { + std::string code; + serialization::readString(file, code); + bool found = false; + + for (uint8_t i = 0; i < getLanguageCount(); i++) { + if (code == LANGUAGE_CODES[i]) { + _language = static_cast(i); + found = true; + break; + } + } + + if (found) { + Serial.printf("[I18N] Loaded language code: %s (%d)\n", code.c_str(), static_cast(_language)); + } else { + Serial.printf("[I18N] Unknown language code in settings: %s\n", code.c_str()); + } + file.close(); return; } - uint8_t lang; - serialization::readPod(file, lang); - if (lang < static_cast(Language::_COUNT)) { - _language = static_cast(lang); - Serial.printf("[I18N] Loaded language: %d\n", static_cast(_language)); + // Legacy migration path: version 1 stored language enum index directly. + if (version == 1) { + uint8_t lang; + serialization::readPod(file, lang); + if (lang < static_cast(Language::_COUNT)) { + _language = static_cast(lang); + Serial.printf("[I18N] Migrating v1 language index: %d -> %s\n", static_cast(_language), + getLanguageCode(_language)); + file.close(); + saveSettings(); + return; + } + file.close(); + Serial.printf("[I18N] Invalid v1 language index: %d\n", static_cast(lang)); + return; } + + Serial.printf("[I18N] Settings version mismatch: %d\n", static_cast(version)); + + file.close(); } // Generate character set for a specific language diff --git a/lib/I18n/I18n.h b/lib/I18n/I18n.h index 347b7fcc..74273b30 100644 --- a/lib/I18n/I18n.h +++ b/lib/I18n/I18n.h @@ -22,6 +22,7 @@ class I18n { Language getLanguage() const { return _language; } void setLanguage(Language lang); + const char* getLanguageCode(Language lang) const; const char* getLanguageName(Language lang) const; void saveSettings(); diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 74d4b197..520be52f 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -15,24 +15,32 @@ Each YAML file must contain: The English file is the reference. Missing keys in other languages are automatically filled from English, with a warning. -Usage: - python gen_i18n.py +By default the script scans the src/ and lib/ trees for STR_* references and +reports any translation keys that are never used. Pass --strip-unused to +omit those keys from the generated output entirely. -Example: +Usage: + python gen_i18n.py [translations_dir [output_dir]] [options] + +Examples: + python gen_i18n.py python gen_i18n.py lib/I18n/translations lib/I18n/ + python gen_i18n.py --strip-unused + python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont """ import sys import os import re from pathlib import Path -from typing import List, Dict, Tuple +from typing import Dict, List, Optional, Set, Tuple # --------------------------------------------------------------------------- # YAML file reading (simple key: "value" format, no PyYAML dependency) # --------------------------------------------------------------------------- + def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str: """ Process escape sequences in a YAML value string. @@ -51,9 +59,7 @@ def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str elif nxt == "n": result.append("\n") else: - raise ValueError( - f"{filepath}:{line_num}: unknown escape '\\{nxt}'" - ) + raise ValueError(f"{filepath}:{line_num}: unknown escape '\\{nxt}'") i += 2 else: result.append(raw[i]) @@ -103,9 +109,11 @@ def parse_yaml_file(filepath: str) -> Dict[str, str]: # Load all languages from a directory of YAML files # --------------------------------------------------------------------------- + def load_translations( translations_dir: str, -) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]]]: + verbose: bool = False, +) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]: """ Read every YAML file in *translations_dir* and return: language_codes e.g. ["EN", "ES", ...] @@ -138,6 +146,31 @@ def load_translations( if english_file is None: raise ValueError("No YAML file with _language_code: EN found") + duplicate_orders: Dict[str, List[str]] = {} + order_to_files: Dict[str, List[str]] = {} + for fname, data in parsed.items(): + order = data.get("_order") + if not order: + continue + order_to_files.setdefault(order, []).append(fname) + + for order, files in order_to_files.items(): + if len(files) > 1: + duplicate_orders[order] = sorted(files) + + if duplicate_orders: + duplicate_messages = [ + f"_order {order}: {', '.join(files)}" + for order, files in sorted( + duplicate_orders.items(), key=lambda item: int(item[0]) + ) + ] + raise ValueError( + "Duplicate _order values found:\n " + + "\n ".join(duplicate_messages) + + "\nEach _order value must be unique to ensure a deterministic language order." + ) + # Order: English first, then by _order metadata (falls back to filename) def sort_key(fname: str) -> Tuple[int, int, str]: """English always first (0), then by _order, then by filename.""" @@ -174,16 +207,20 @@ def load_translations( raise ValueError(f"Invalid C++ identifier in English file: '{key}'") # Build translations dict, filling missing keys from English + inherited_sets: List[Set[str]] = [set() for _ in ordered_files] translations: Dict[str, List[str]] = {} for key in string_keys: row: List[str] = [] - for fname in ordered_files: + for lang_idx, fname in enumerate(ordered_files): data = parsed[fname] value = data.get(key, "") if not value.strip() and fname != english_file: value = english_data[key] - lang_code = parsed[fname].get("_language_code", fname) - print(f" INFO: '{key}' missing in {lang_code}, using English fallback") + inherited_sets[lang_idx].add(key) + if verbose: + print( + f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback" + ) row.append(value) translations[key] = row @@ -195,45 +232,72 @@ def load_translations( extra = [k for k in data if not k.startswith("_") and k not in english_data] if extra: lang_code = data.get("_language_code", fname) - print(f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}") + if verbose: + print( + f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}" + ) - print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") - return language_codes, language_names, string_keys, translations + if verbose: + print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") + return language_codes, language_names, string_keys, translations, inherited_sets + + +# --------------------------------------------------------------------------- +# Unused-string detection +# --------------------------------------------------------------------------- + +_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"} + + +def find_used_string_keys( + src_dirs: List[str], + skip_filenames: Optional[Set[str]] = None, +) -> Set[str]: + """ + Scan C/C++ source files under *src_dirs* for STR_* identifiers. + + Files whose basename appears in *skip_filenames* are skipped so that + the generated I18n files don't count as "usage" of themselves. + + Returns the set of all STR_KEY names that appear at least once. + """ + if skip_filenames is None: + skip_filenames = _GENERATED_FILENAMES + + pattern = re.compile(r"\bSTR_[A-Za-z0-9_]+\b") + used: Set[str] = set() + + for src_dir in src_dirs: + p = Path(src_dir) + if not p.is_dir(): + continue + for f in p.rglob("*"): + if f.suffix not in {".cpp", ".h", ".c"}: + continue + if f.name in skip_filenames: + continue + try: + text = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for m in pattern.finditer(text): + used.add(m.group(0)) + + return used + + +def report_unused_keys( + string_keys: List[str], + used_keys: Set[str], +) -> List[str]: + """Return a sorted list of keys from *string_keys* absent in *used_keys*.""" + return [k for k in sorted(string_keys) if k not in used_keys] # --------------------------------------------------------------------------- # C++ string escaping # --------------------------------------------------------------------------- -LANG_ABBREVIATIONS = { - "english": "EN", - "español": "ES", "espanol": "ES", - "italiano": "IT", - "svenska": "SV", - "français": "FR", "francais": "FR", - "deutsch": "DE", "german": "DE", - "polski": "PL", - "português": "PT", "portugues": "PT", "português (brasil)": "PO", - "中文": "ZH", "chinese": "ZH", - "日本語": "JA", "japanese": "JA", - "한국어": "KO", "korean": "KO", - "русский": "RU", "russian": "RU", - "العربية": "AR", "arabic": "AR", - "עברית": "HE", "hebrew": "HE", - "فارسی": "FA", "persian": "FA", - "čeština": "CS", - "türkçe": "TR", "turkish": "TR", - "Қазақша": "KK", "kazakh": "KK", -} - - -def get_lang_abbreviation(lang_code: str, lang_name: str) -> str: - """Return a 2-letter abbreviation for a language.""" - lower = lang_name.lower() - if lower in LANG_ABBREVIATIONS: - return LANG_ABBREVIATIONS[lower] - return lang_code[:2].upper() - def escape_cpp_string(s: str) -> List[str]: r""" @@ -267,12 +331,12 @@ def escape_cpp_string(s: str) -> List[str]: if ch == "\\" and i + 1 < len(s): nxt = s[i + 1] - if nxt in "ntr\"\\": + if nxt in 'ntr"\\': current.append(ch + nxt) i += 2 elif nxt == "x" and i + 3 < len(s): current.append(s[i : i + 4]) - _flush() # segment break after hex + _flush() # segment break after hex i += 4 else: current.append("\\\\") @@ -286,7 +350,7 @@ def escape_cpp_string(s: str) -> List[str]: else: for byte in ch.encode("utf-8"): current.append(f"\\x{byte:02X}") - _flush() # segment break after hex + _flush() # segment break after hex i += 1 # Flush remaining content @@ -321,11 +385,11 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List last_space = -1 idx = 0 while idx <= MAX_CONTENT_LEN and idx < len(current): - if current[idx] == ' ': + if current[idx] == " ": last_space = idx # Handle escapes to step correctly - if current[idx] == '\\': + if current[idx] == "\\": idx += 2 else: idx += 1 @@ -340,7 +404,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List # No space, forced break at MAX_CONTENT_LEN (or slightly less) cut_at = MAX_CONTENT_LEN # Don't cut in the middle of an escape sequence - if current[cut_at - 1] == '\\': + if current[cut_at - 1] == "\\": cut_at -= 1 lines.append(f'{indent}"{current[:cut_at]}"') @@ -356,6 +420,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List # Character-set computation # --------------------------------------------------------------------------- + def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str: """Return a sorted string of every unique character used in a language.""" chars = set() @@ -369,11 +434,13 @@ def compute_character_set(translations: Dict[str, List[str]], lang_index: int) - # Code generators # --------------------------------------------------------------------------- + def generate_keys_header( languages: List[str], language_names: List[str], string_keys: List[str], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nKeys.h.""" lines: List[str] = [ @@ -381,14 +448,15 @@ def generate_keys_header( "#include ", "", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", + "// clang-format off", "", - "// Forward declaration for string arrays", + "// Forward declarations for flat string data blobs and offset tables", "namespace i18n_strings {", ] - for code, name in zip(languages, language_names): - abbrev = get_lang_abbreviation(code, name) - lines.append(f"extern const char* const STRINGS_{abbrev}[];") + for code in languages: + lines.append(f"extern const char STRINGS_{code}_DATA[];") + lines.append(f"extern const uint16_t OFFSETS_{code}[];") lines.append("} // namespace i18n_strings") lines.append("") @@ -403,6 +471,10 @@ def generate_keys_header( lines.append("") # Extern declarations + lines.append("// Language codes (defined in I18nStrings.cpp)") + lines.append("extern const char* const LANGUAGE_CODES[];") + lines.append("") + lines.append("// Language display names (defined in I18nStrings.cpp)") lines.append("extern const char* const LANGUAGE_NAMES[];") lines.append("") @@ -420,17 +492,28 @@ def generate_keys_header( lines.append("};") lines.append("") - # getStringArray helper - lines.append("// Helper function to get string array for a language") - lines.append("inline const char* const* getStringArray(Language lang) {") + # LangStrings struct + lines.append("// Holds a flat string blob and its offset table for one language") + lines.append("struct LangStrings {") + lines.append(" const char* data;") + lines.append(" const uint16_t* offsets;") + lines.append("};") + lines.append("") + + # getLanguageStrings helper + lines.append("// Helper function to get string data for a language") + lines.append("inline LangStrings getLanguageStrings(Language lang) {") lines.append(" switch (lang) {") - for code, name in zip(languages, language_names): - abbrev = get_lang_abbreviation(code, name) + for code in languages: lines.append(f" case Language::{code}:") - lines.append(f" return i18n_strings::STRINGS_{abbrev};") - first_abbrev = get_lang_abbreviation(languages[0], language_names[0]) + lines.append( + f" return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};" + ) + first_code = languages[0] lines.append(" default:") - lines.append(f" return i18n_strings::STRINGS_{first_abbrev};") + lines.append( + f" return {{i18n_strings::STRINGS_{first_code}_DATA, i18n_strings::OFFSETS_{first_code}}};" + ) lines.append(" }") lines.append("}") lines.append("") @@ -451,9 +534,9 @@ def generate_keys_header( key=lambda i: languages[i], ) sorted_indices = [english_idx] + rest - comment_names = ", ".join(language_names[i] for i in sorted_indices) lines.append("// Sorted language indices by code (auto-generated by gen_i18n.py)") - lines.append(f"// Order: {comment_names}") + for rank, idx in enumerate(sorted_indices): + lines.append(f"// {rank:>2}: {languages[idx]:<4} {language_names[idx]}") lines.append( "constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {" f"{', '.join(str(i) for i in sorted_indices)}" @@ -463,38 +546,36 @@ def generate_keys_header( lines.append( "static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount()," ) - lines.append( - ' "SORTED_LANGUAGE_INDICES size mismatch");' - ) + lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");') - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) def generate_strings_header( languages: List[str], language_names: List[str], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nStrings.h.""" lines: List[str] = [ "#pragma once", - '#include ', - "", '#include "I18nKeys.h"', "", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", + "// clang-format off", "", "namespace i18n_strings {", "", ] - for code, name in zip(languages, language_names): - abbrev = get_lang_abbreviation(code, name) - lines.append(f"extern const char* const STRINGS_{abbrev}[];") + for code in languages: + lines.append(f"extern const char STRINGS_{code}_DATA[];") + lines.append(f"extern const uint16_t OFFSETS_{code}[];") lines.append("") lines.append("} // namespace i18n_strings") - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) def generate_strings_cpp( @@ -503,15 +584,26 @@ def generate_strings_cpp( string_keys: List[str], translations: Dict[str, List[str]], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nStrings.cpp.""" lines: List[str] = [ + "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", + "// clang-format off", '#include "I18nStrings.h"', "", - "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", + "#include ", "", ] + # LANGUAGE_NAMES array + lines.append("// Language codes") + lines.append("const char* const LANGUAGE_CODES[] = {") + for code in languages: + _append_string_entry(lines, code) + lines.append("};") + lines.append("") + # LANGUAGE_NAMES array lines.append("// Language display names") lines.append("const char* const LANGUAGE_NAMES[] = {") @@ -529,18 +621,38 @@ def generate_strings_cpp( lines.append("};") lines.append("") - # Per-language string arrays + # Per-language flat string blobs and offset tables lines.append("namespace i18n_strings {") lines.append("") - for lang_idx, (code, name) in enumerate(zip(languages, language_names)): - abbrev = get_lang_abbreviation(code, name) - lines.append(f"const char* const STRINGS_{abbrev}[] = {{") + for lang_idx, code in enumerate(languages): + lang_strings = [translations[key][lang_idx] for key in string_keys] - for key in string_keys: - text = translations[key][lang_idx] - _append_string_entry(lines, text) + # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator) + offsets: List[int] = [] + current_offset = 0 + for s in lang_strings: + offsets.append(current_offset) + current_offset += len(s.encode("utf-8")) + 1 + if current_offset > 65535: + raise ValueError( + f"Language {code}: total string data ({current_offset} bytes) " + "exceeds uint16_t offset range (65535)" + ) + # Flat string data blob — all strings concatenated with \0 separators. + lines.append(f"const char STRINGS_{code}_DATA[] =") + for text in lang_strings: + _append_string_data_entry(lines, text) + lines.append(";") + lines.append("") + + # Offset table — one uint16_t per StrId + lines.append(f"const uint16_t OFFSETS_{code}[] = {{") + chunk_size = 12 + for i in range(0, len(offsets), chunk_size): + chunk = offsets[i : i + chunk_size] + lines.append(" " + ", ".join(str(o) for o in chunk) + ",") lines.append("};") lines.append("") @@ -549,25 +661,108 @@ def generate_strings_cpp( # Compile-time size checks lines.append("// Compile-time validation of array sizes") - for code, name in zip(languages, language_names): - abbrev = get_lang_abbreviation(code, name) + for code in languages: lines.append( - f"static_assert(sizeof(i18n_strings::STRINGS_{abbrev}) " - f"/ sizeof(i18n_strings::STRINGS_{abbrev}[0]) ==" + f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) " + f"/ sizeof(i18n_strings::OFFSETS_{code}[0]) ==" ) lines.append(" static_cast(StrId::_COUNT),") - lines.append(f' "STRINGS_{abbrev} size mismatch");') + lines.append(f' "OFFSETS_{code} size mismatch");') - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _append_string_entry( - lines: List[str], text: str, comment: str = "" + +def _print_language_table( + language_codes: List[str], + language_names: List[str], + inherited_sets: List[Set[str]], + string_keys: List[str], + unused_keys: Set[str], + data_sizes: List[int], ) -> None: + """Print a per-language summary table.""" + total = len(string_keys) + headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)") + + rows = [] + for code, name, inherited, size in zip( + language_codes, language_names, inherited_sets, data_sizes + ): + own = total - len(inherited) + fallback = len(inherited) + # strings this language translated but the code never calls + unused = len(unused_keys - inherited) + rows.append((name, code, str(own), str(fallback), str(unused), str(size))) + + # EN first, then alphabetically by ISO code + rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1])) + + col_widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + col_widths[i] = max(col_widths[i], len(cell)) + + fmt = " ".join(f"{{:<{w}}}" for w in col_widths) + sep = " ".join("-" * w for w in col_widths) + + def _safe_print(line: str) -> None: + print( + line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode( + sys.stdout.encoding or "utf-8", errors="replace" + ) + ) + + _safe_print(fmt.format(*headers)) + _safe_print(sep) + for row in rows: + _safe_print(fmt.format(*row)) + used = total - len(unused_keys) + total_size = sum(data_sizes) + n_lang = len(rows) + n_keys = len(string_keys) + # Current layout: uint16_t offset table (2 B per string per language) + offset_table_size = n_lang * n_keys * 2 + current_total = total_size + offset_table_size + # Previous layout: const char* pointer array (4 B per string per language) + old_pointer_table_size = n_lang * n_keys * 4 + old_total = total_size + old_pointer_table_size + saved = old_total - current_total + print( + f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}" + ) + print( + f" Flash (now): {total_size:>7,} B strings + {offset_table_size:>6,} B offset tables (uint16_t)" + f" = {current_total:>7,} B" + ) + print( + f" Flash (before): {total_size:>7,} B strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)" + f" = {old_total:>7,} B" + ) + print(f" Saved by offset tables: {saved:,} B") + + +def _append_string_data_entry(lines: List[str], text: str) -> None: + """ + Escape *text*, append a \\0 null separator, and format as indented C++ + string literal lines for inclusion in a flat char data array blob. + """ + segments = escape_cpp_string(text) + # Append the null entry separator to the last segment + if segments and segments[-1] != "": + segments[-1] += "\\0" + elif segments: + segments[-1] = "\\0" + else: + segments = ["\\0"] + lines.extend(format_cpp_string_literal(segments)) + + +def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None: """Escape *text*, format as indented C++ lines, append comma (and optional comment).""" segments = escape_cpp_string(text) formatted = format_cpp_string_literal(segments) @@ -576,21 +771,30 @@ def _append_string_entry( lines.extend(formatted) -def _write_file(path: str, lines: List[str]) -> None: +def _write_file(path: str, lines: List[str], verbose: bool = False) -> None: with open(path, "w", encoding="utf-8", newline="\n") as f: f.write("\n".join(lines)) f.write("\n") - print(f"Generated: {path}") + if verbose: + print(f"Generated: {path}") # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -def main(translations_dir=None, output_dir=None) -> None: + +def main( + translations_dir: Optional[str] = None, + output_dir: Optional[str] = None, + src_dirs: Optional[List[str]] = None, + strip_unused: bool = False, + verbose: bool = False, +) -> None: # Default paths (relative to project root) default_translations_dir = "lib/I18n/translations" default_output_dir = "lib/I18n/" + default_src_dirs = ["src", "lib"] if translations_dir is None or output_dir is None: if len(sys.argv) == 3: @@ -601,6 +805,8 @@ def main(translations_dir=None, output_dir=None) -> None: translations_dir = default_translations_dir output_dir = default_output_dir + if src_dirs is None: + src_dirs = default_src_dirs if not os.path.isdir(translations_dir): print(f"Error: Translations directory not found: {translations_dir}") @@ -610,26 +816,91 @@ def main(translations_dir=None, output_dir=None) -> None: print(f"Error: Output directory not found: {output_dir}") sys.exit(1) - print(f"Reading translations from: {translations_dir}") - print(f"Output directory: {output_dir}") - print() + if verbose: + print(f"Reading translations from: {translations_dir}") + print(f"Output directory: {output_dir}") + print() try: - languages, language_names, string_keys, translations = load_translations( - translations_dir + languages, language_names, string_keys, translations, inherited_sets = ( + load_translations(translations_dir, verbose) ) + # --- Unused-string detection --- + scan_dirs = [d for d in src_dirs if os.path.isdir(d)] + if scan_dirs: + used_keys = find_used_string_keys(scan_dirs) + unused_set = set(report_unused_keys(string_keys, used_keys)) + else: + used_keys = set(string_keys) + unused_set = set() + + # --- Missing-string detection (used in code but absent from English) --- + missing_keys = sorted(used_keys - set(string_keys)) + if missing_keys: + print( + f"\n CRITICAL: {len(missing_keys)} string(s) used in source but missing from english.yaml:" + ) + for key in missing_keys: + print(f" - {key}") + print() + sys.exit(1) + + # Compute per-language data blob sizes: + # sum of UTF-8 byte length + 1 (null terminator) per string + data_sizes = [ + sum(len(translations[k][i].encode("utf-8")) + 1 for k in string_keys) + for i in range(len(languages)) + ] + + _print_language_table( + languages, + language_names, + inherited_sets, + string_keys, + unused_set, + data_sizes, + ) + print() + + if verbose and unused_set: + print(f" Unused keys ({len(unused_set)}):") + for key in sorted(unused_set): + print(f" - {key}") + print() + + if unused_set and strip_unused: + string_keys = [k for k in string_keys if k not in unused_set] + translations = { + k: v for k, v in translations.items() if k not in unused_set + } + inherited_sets = [s - unused_set for s in inherited_sets] + print(f" Stripping {len(unused_set)} unused string(s) from output.") + out = Path(output_dir) - generate_keys_header(languages, language_names, string_keys, str(out / "I18nKeys.h")) - generate_strings_header(languages, language_names, str(out / "I18nStrings.h")) + generate_keys_header( + languages, language_names, string_keys, str(out / "I18nKeys.h"), verbose + ) + generate_strings_header( + languages, language_names, str(out / "I18nStrings.h"), verbose + ) generate_strings_cpp( - languages, language_names, string_keys, translations, str(out / "I18nStrings.cpp") + languages, + language_names, + string_keys, + translations, + str(out / "I18nStrings.cpp"), + verbose, ) print() - print("✓ Code generation complete!") + print("Code generation complete!") print(f" Languages: {len(languages)}") print(f" String keys: {len(string_keys)}") + if unused_set and not strip_unused: + print( + f" Unused keys: {len(unused_set)} (pass --strip-unused to remove them)" + ) except Exception as e: print(f"\nError: {e}") @@ -637,11 +908,52 @@ def main(translations_dir=None, output_dir=None) -> None: if __name__ == "__main__": - main() + import argparse + + parser = argparse.ArgumentParser( + description="Generate I18n C++ files from per-language YAML translations." + ) + parser.add_argument( + "translations_dir", + nargs="?", + default=None, + help="Path to the translations directory (default: lib/I18n/translations)", + ) + parser.add_argument( + "output_dir", + nargs="?", + default=None, + help="Path to the output directory (default: lib/I18n/)", + ) + parser.add_argument( + "--src-dirs", + nargs="+", + metavar="DIR", + default=None, + help="Source directories to scan for STR_* usage (default: src lib)", + ) + parser.add_argument( + "--strip-unused", + action="store_true", + help="Remove unused STR_* keys from the generated output", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print per-key INFO/WARNING messages and file generation details", + ) + args = parser.parse_args() + main( + args.translations_dir, + args.output_dir, + args.src_dirs, + args.strip_unused, + args.verbose, + ) else: try: Import("env") - print("Running i18n generation script from PlatformIO...") - main() + main(strip_unused=True) except NameError: pass