diff --git a/lib/I18n/I18n.cpp b/lib/I18n/I18n.cpp index 55cf8fc4..c77fce9a 100644 --- a/lib/I18n/I18n.cpp +++ b/lib/I18n/I18n.cpp @@ -24,8 +24,8 @@ const char* I18n::get(StrId id) const { } // Use generated helper function - no hardcoded switch needed! - const char* const* strings = getStringArray(_language); - return strings[index]; + const LangStrings lang = getLanguageStrings(_language); + return lang.data + lang.offsets[index]; } void I18n::setLanguage(Language lang) { diff --git a/scripts/gen_i18n.py b/scripts/gen_i18n.py index 74d4b197..5d553427 100755 --- a/scripts/gen_i18n.py +++ b/scripts/gen_i18n.py @@ -15,24 +15,32 @@ Each YAML file must contain: The English file is the reference. Missing keys in other languages are automatically filled from English, with a warning. -Usage: - python gen_i18n.py +By default the script scans the src/ and lib/ trees for STR_* references and +reports any translation keys that are never used. Pass --strip-unused to +omit those keys from the generated output entirely. -Example: +Usage: + python gen_i18n.py [translations_dir [output_dir]] [options] + +Examples: + python gen_i18n.py python gen_i18n.py lib/I18n/translations lib/I18n/ + python gen_i18n.py --strip-unused + python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont """ import sys import os import re from pathlib import Path -from typing import List, Dict, Tuple +from typing import Dict, List, Optional, Set, Tuple # --------------------------------------------------------------------------- # YAML file reading (simple key: "value" format, no PyYAML dependency) # --------------------------------------------------------------------------- + def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str: """ Process escape sequences in a YAML value string. @@ -51,9 +59,7 @@ def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str elif nxt == "n": result.append("\n") else: - raise ValueError( - f"{filepath}:{line_num}: unknown escape '\\{nxt}'" - ) + raise ValueError(f"{filepath}:{line_num}: unknown escape '\\{nxt}'") i += 2 else: result.append(raw[i]) @@ -103,9 +109,11 @@ def parse_yaml_file(filepath: str) -> Dict[str, str]: # Load all languages from a directory of YAML files # --------------------------------------------------------------------------- + def load_translations( translations_dir: str, -) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]]]: + verbose: bool = False, +) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]: """ Read every YAML file in *translations_dir* and return: language_codes e.g. ["EN", "ES", ...] @@ -174,16 +182,20 @@ def load_translations( raise ValueError(f"Invalid C++ identifier in English file: '{key}'") # Build translations dict, filling missing keys from English + inherited_sets: List[Set[str]] = [set() for _ in ordered_files] translations: Dict[str, List[str]] = {} for key in string_keys: row: List[str] = [] - for fname in ordered_files: + for lang_idx, fname in enumerate(ordered_files): data = parsed[fname] value = data.get(key, "") if not value.strip() and fname != english_file: value = english_data[key] - lang_code = parsed[fname].get("_language_code", fname) - print(f" INFO: '{key}' missing in {lang_code}, using English fallback") + inherited_sets[lang_idx].add(key) + if verbose: + print( + f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback" + ) row.append(value) translations[key] = row @@ -195,10 +207,66 @@ def load_translations( extra = [k for k in data if not k.startswith("_") and k not in english_data] if extra: lang_code = data.get("_language_code", fname) - print(f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}") + if verbose: + print( + f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}" + ) - print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") - return language_codes, language_names, string_keys, translations + if verbose: + print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") + return language_codes, language_names, string_keys, translations, inherited_sets + + +# --------------------------------------------------------------------------- +# Unused-string detection +# --------------------------------------------------------------------------- + +_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"} + + +def find_used_string_keys( + src_dirs: List[str], + skip_filenames: Optional[Set[str]] = None, +) -> Set[str]: + """ + Scan C/C++ source files under *src_dirs* for STR_* identifiers. + + Files whose basename appears in *skip_filenames* are skipped so that + the generated I18n files don't count as "usage" of themselves. + + Returns the set of all STR_KEY names that appear at least once. + """ + if skip_filenames is None: + skip_filenames = _GENERATED_FILENAMES + + pattern = re.compile(r"\bSTR_[A-Z0-9_]+\b") + used: Set[str] = set() + + for src_dir in src_dirs: + p = Path(src_dir) + if not p.is_dir(): + continue + for f in p.rglob("*"): + if f.suffix not in {".cpp", ".h", ".c"}: + continue + if f.name in skip_filenames: + continue + try: + text = f.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + for m in pattern.finditer(text): + used.add(m.group(0)) + + return used + + +def report_unused_keys( + string_keys: List[str], + used_keys: Set[str], +) -> List[str]: + """Return a sorted list of keys from *string_keys* absent in *used_keys*.""" + return [k for k in string_keys if k not in used_keys] # --------------------------------------------------------------------------- @@ -207,23 +275,37 @@ def load_translations( LANG_ABBREVIATIONS = { "english": "EN", - "español": "ES", "espanol": "ES", + "español": "ES", + "espanol": "ES", "italiano": "IT", "svenska": "SV", - "français": "FR", "francais": "FR", - "deutsch": "DE", "german": "DE", + "français": "FR", + "francais": "FR", + "deutsch": "DE", + "german": "DE", "polski": "PL", - "português": "PT", "portugues": "PT", "português (brasil)": "PO", - "中文": "ZH", "chinese": "ZH", - "日本語": "JA", "japanese": "JA", - "한국어": "KO", "korean": "KO", - "русский": "RU", "russian": "RU", - "العربية": "AR", "arabic": "AR", - "עברית": "HE", "hebrew": "HE", - "فارسی": "FA", "persian": "FA", + "português": "PT", + "portugues": "PT", + "português (brasil)": "PO", + "中文": "ZH", + "chinese": "ZH", + "日本語": "JA", + "japanese": "JA", + "한국어": "KO", + "korean": "KO", + "русский": "RU", + "russian": "RU", + "العربية": "AR", + "arabic": "AR", + "עברית": "HE", + "hebrew": "HE", + "فارسی": "FA", + "persian": "FA", "čeština": "CS", - "türkçe": "TR", "turkish": "TR", - "Қазақша": "KK", "kazakh": "KK", + "türkçe": "TR", + "turkish": "TR", + "Қазақша": "KK", + "kazakh": "KK", } @@ -267,12 +349,12 @@ def escape_cpp_string(s: str) -> List[str]: if ch == "\\" and i + 1 < len(s): nxt = s[i + 1] - if nxt in "ntr\"\\": + if nxt in 'ntr"\\': current.append(ch + nxt) i += 2 elif nxt == "x" and i + 3 < len(s): current.append(s[i : i + 4]) - _flush() # segment break after hex + _flush() # segment break after hex i += 4 else: current.append("\\\\") @@ -286,7 +368,7 @@ def escape_cpp_string(s: str) -> List[str]: else: for byte in ch.encode("utf-8"): current.append(f"\\x{byte:02X}") - _flush() # segment break after hex + _flush() # segment break after hex i += 1 # Flush remaining content @@ -321,11 +403,11 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List last_space = -1 idx = 0 while idx <= MAX_CONTENT_LEN and idx < len(current): - if current[idx] == ' ': + if current[idx] == " ": last_space = idx # Handle escapes to step correctly - if current[idx] == '\\': + if current[idx] == "\\": idx += 2 else: idx += 1 @@ -340,7 +422,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List # No space, forced break at MAX_CONTENT_LEN (or slightly less) cut_at = MAX_CONTENT_LEN # Don't cut in the middle of an escape sequence - if current[cut_at - 1] == '\\': + if current[cut_at - 1] == "\\": cut_at -= 1 lines.append(f'{indent}"{current[:cut_at]}"') @@ -356,6 +438,7 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List # Character-set computation # --------------------------------------------------------------------------- + def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str: """Return a sorted string of every unique character used in a language.""" chars = set() @@ -369,11 +452,13 @@ def compute_character_set(translations: Dict[str, List[str]], lang_index: int) - # Code generators # --------------------------------------------------------------------------- + def generate_keys_header( languages: List[str], language_names: List[str], string_keys: List[str], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nKeys.h.""" lines: List[str] = [ @@ -382,13 +467,14 @@ def generate_keys_header( "", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", "", - "// Forward declaration for string arrays", + "// Forward declarations for flat string data blobs and offset tables", "namespace i18n_strings {", ] for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) - lines.append(f"extern const char* const STRINGS_{abbrev}[];") + lines.append(f"extern const char STRINGS_{abbrev}_DATA[];") + lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];") lines.append("} // namespace i18n_strings") lines.append("") @@ -420,17 +506,29 @@ def generate_keys_header( lines.append("};") lines.append("") - # getStringArray helper - lines.append("// Helper function to get string array for a language") - lines.append("inline const char* const* getStringArray(Language lang) {") + # LangStrings struct + lines.append("// Holds a flat string blob and its offset table for one language") + lines.append("struct LangStrings {") + lines.append(" const char* data;") + lines.append(" const uint16_t* offsets;") + lines.append("};") + lines.append("") + + # getLanguageStrings helper + lines.append("// Helper function to get string data for a language") + lines.append("inline LangStrings getLanguageStrings(Language lang) {") lines.append(" switch (lang) {") for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) lines.append(f" case Language::{code}:") - lines.append(f" return i18n_strings::STRINGS_{abbrev};") + lines.append( + f" return {{i18n_strings::STRINGS_{abbrev}_DATA, i18n_strings::OFFSETS_{abbrev}}};" + ) first_abbrev = get_lang_abbreviation(languages[0], language_names[0]) lines.append(" default:") - lines.append(f" return i18n_strings::STRINGS_{first_abbrev};") + lines.append( + f" return {{i18n_strings::STRINGS_{first_abbrev}_DATA, i18n_strings::OFFSETS_{first_abbrev}}};" + ) lines.append(" }") lines.append("}") lines.append("") @@ -463,22 +561,21 @@ def generate_keys_header( lines.append( "static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount()," ) - lines.append( - ' "SORTED_LANGUAGE_INDICES size mismatch");' - ) + lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");') - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) def generate_strings_header( languages: List[str], language_names: List[str], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nStrings.h.""" lines: List[str] = [ "#pragma once", - '#include ', + "#include ", "", '#include "I18nKeys.h"', "", @@ -490,11 +587,12 @@ def generate_strings_header( for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) - lines.append(f"extern const char* const STRINGS_{abbrev}[];") + lines.append(f"extern const char STRINGS_{abbrev}_DATA[];") + lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];") lines.append("") lines.append("} // namespace i18n_strings") - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) def generate_strings_cpp( @@ -503,6 +601,7 @@ def generate_strings_cpp( string_keys: List[str], translations: Dict[str, List[str]], output_path: str, + verbose: bool = False, ) -> None: """Generate I18nStrings.cpp.""" lines: List[str] = [ @@ -529,18 +628,42 @@ def generate_strings_cpp( lines.append("};") lines.append("") - # Per-language string arrays + # Per-language flat string blobs and offset tables lines.append("namespace i18n_strings {") lines.append("") for lang_idx, (code, name) in enumerate(zip(languages, language_names)): abbrev = get_lang_abbreviation(code, name) - lines.append(f"const char* const STRINGS_{abbrev}[] = {{") + lang_strings = [translations[key][lang_idx] for key in string_keys] - for key in string_keys: - text = translations[key][lang_idx] - _append_string_entry(lines, text) + # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator) + offsets: List[int] = [] + current_offset = 0 + for s in lang_strings: + offsets.append(current_offset) + current_offset += len(s.encode("utf-8")) + 1 + if current_offset > 65535: + raise ValueError( + f"Language {code}: total string data ({current_offset} bytes) " + "exceeds uint16_t offset range (65535)" + ) + # Flat string data blob — all strings concatenated with \0 separators. + # clang-format off/on avoids reformatting of the adjacent string literals. + lines.append("// clang-format off") + lines.append(f"const char STRINGS_{abbrev}_DATA[] =") + for text in lang_strings: + _append_string_data_entry(lines, text) + lines.append(";") + lines.append("// clang-format on") + lines.append("") + + # Offset table — one uint16_t per StrId + lines.append(f"const uint16_t OFFSETS_{abbrev}[] = {{") + chunk_size = 12 + for i in range(0, len(offsets), chunk_size): + chunk = offsets[i : i + chunk_size] + lines.append(" " + ", ".join(str(o) for o in chunk) + ",") lines.append("};") lines.append("") @@ -552,22 +675,84 @@ def generate_strings_cpp( for code, name in zip(languages, language_names): abbrev = get_lang_abbreviation(code, name) lines.append( - f"static_assert(sizeof(i18n_strings::STRINGS_{abbrev}) " - f"/ sizeof(i18n_strings::STRINGS_{abbrev}[0]) ==" + f"static_assert(sizeof(i18n_strings::OFFSETS_{abbrev}) " + f"/ sizeof(i18n_strings::OFFSETS_{abbrev}[0]) ==" ) lines.append(" static_cast(StrId::_COUNT),") - lines.append(f' "STRINGS_{abbrev} size mismatch");') + lines.append(f' "OFFSETS_{abbrev} size mismatch");') - _write_file(output_path, lines) + _write_file(output_path, lines, verbose) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _append_string_entry( - lines: List[str], text: str, comment: str = "" + +def _print_language_table( + language_codes: List[str], + language_names: List[str], + inherited_sets: List[Set[str]], + string_keys: List[str], + unused_keys: Set[str], ) -> None: + """Print a per-language summary table.""" + total = len(string_keys) + headers = ("Language", "Code", "Own", "Fallback", "Unused") + + rows = [] + for code, name, inherited in zip(language_codes, language_names, inherited_sets): + own = total - len(inherited) + fallback = len(inherited) + # strings this language translated but the code never calls + unused = len(unused_keys - inherited) + rows.append((name, code, str(own), str(fallback), str(unused))) + + # EN first, then alphabetically by ISO code + rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1])) + + col_widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + col_widths[i] = max(col_widths[i], len(cell)) + + fmt = " ".join(f"{{:<{w}}}" for w in col_widths) + sep = " ".join("-" * w for w in col_widths) + + def _safe_print(line: str) -> None: + print( + line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode( + sys.stdout.encoding or "utf-8", errors="replace" + ) + ) + + _safe_print(fmt.format(*headers)) + _safe_print(sep) + for row in rows: + _safe_print(fmt.format(*row)) + used = total - len(unused_keys) + print( + f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}" + ) + + +def _append_string_data_entry(lines: List[str], text: str) -> None: + """ + Escape *text*, append a \\0 null separator, and format as indented C++ + string literal lines for inclusion in a flat char data array blob. + """ + segments = escape_cpp_string(text) + # Append the null entry separator to the last segment + if segments and segments[-1] != "": + segments[-1] += "\\0" + elif segments: + segments[-1] = "\\0" + else: + segments = ["\\0"] + lines.extend(format_cpp_string_literal(segments)) + + +def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None: """Escape *text*, format as indented C++ lines, append comma (and optional comment).""" segments = escape_cpp_string(text) formatted = format_cpp_string_literal(segments) @@ -576,21 +761,30 @@ def _append_string_entry( lines.extend(formatted) -def _write_file(path: str, lines: List[str]) -> None: +def _write_file(path: str, lines: List[str], verbose: bool = False) -> None: with open(path, "w", encoding="utf-8", newline="\n") as f: f.write("\n".join(lines)) f.write("\n") - print(f"Generated: {path}") + if verbose: + print(f"Generated: {path}") # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -def main(translations_dir=None, output_dir=None) -> None: + +def main( + translations_dir: Optional[str] = None, + output_dir: Optional[str] = None, + src_dirs: Optional[List[str]] = None, + strip_unused: bool = False, + verbose: bool = False, +) -> None: # Default paths (relative to project root) default_translations_dir = "lib/I18n/translations" default_output_dir = "lib/I18n/" + default_src_dirs = ["src", "lib"] if translations_dir is None or output_dir is None: if len(sys.argv) == 3: @@ -601,6 +795,8 @@ def main(translations_dir=None, output_dir=None) -> None: translations_dir = default_translations_dir output_dir = default_output_dir + if src_dirs is None: + src_dirs = default_src_dirs if not os.path.isdir(translations_dir): print(f"Error: Translations directory not found: {translations_dir}") @@ -610,26 +806,68 @@ def main(translations_dir=None, output_dir=None) -> None: print(f"Error: Output directory not found: {output_dir}") sys.exit(1) - print(f"Reading translations from: {translations_dir}") - print(f"Output directory: {output_dir}") - print() + if verbose: + print(f"Reading translations from: {translations_dir}") + print(f"Output directory: {output_dir}") + print() try: - languages, language_names, string_keys, translations = load_translations( - translations_dir + languages, language_names, string_keys, translations, inherited_sets = ( + load_translations(translations_dir, verbose) ) + # --- Unused-string detection --- + scan_dirs = [d for d in src_dirs if os.path.isdir(d)] + if scan_dirs: + used_keys = find_used_string_keys(scan_dirs) + unused_set = set(report_unused_keys(string_keys, used_keys)) + else: + used_keys = set(string_keys) + unused_set = set() + + _print_language_table( + languages, language_names, inherited_sets, string_keys, unused_set + ) + print() + + if verbose and unused_set: + print(f" Unused keys ({len(unused_set)}):") + for key in sorted(unused_set): + print(f" - {key}") + print() + + if unused_set and strip_unused: + string_keys = [k for k in string_keys if k not in unused_set] + translations = { + k: v for k, v in translations.items() if k not in unused_set + } + inherited_sets = [s - unused_set for s in inherited_sets] + print(f" Stripping {len(unused_set)} unused string(s) from output.") + out = Path(output_dir) - generate_keys_header(languages, language_names, string_keys, str(out / "I18nKeys.h")) - generate_strings_header(languages, language_names, str(out / "I18nStrings.h")) + generate_keys_header( + languages, language_names, string_keys, str(out / "I18nKeys.h"), verbose + ) + generate_strings_header( + languages, language_names, str(out / "I18nStrings.h"), verbose + ) generate_strings_cpp( - languages, language_names, string_keys, translations, str(out / "I18nStrings.cpp") + languages, + language_names, + string_keys, + translations, + str(out / "I18nStrings.cpp"), + verbose, ) print() - print("✓ Code generation complete!") + print("Code generation complete!") print(f" Languages: {len(languages)}") print(f" String keys: {len(string_keys)}") + if unused_set and not strip_unused: + print( + f" Unused keys: {len(unused_set)} (pass --strip-unused to remove them)" + ) except Exception as e: print(f"\nError: {e}") @@ -637,11 +875,52 @@ def main(translations_dir=None, output_dir=None) -> None: if __name__ == "__main__": - main() + import argparse + + parser = argparse.ArgumentParser( + description="Generate I18n C++ files from per-language YAML translations." + ) + parser.add_argument( + "translations_dir", + nargs="?", + default=None, + help="Path to the translations directory (default: lib/I18n/translations)", + ) + parser.add_argument( + "output_dir", + nargs="?", + default=None, + help="Path to the output directory (default: lib/I18n/)", + ) + parser.add_argument( + "--src-dirs", + nargs="+", + metavar="DIR", + default=None, + help="Source directories to scan for STR_* usage (default: src lib)", + ) + parser.add_argument( + "--strip-unused", + action="store_true", + help="Remove unused STR_* keys from the generated output", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print per-key INFO/WARNING messages and file generation details", + ) + args = parser.parse_args() + main( + args.translations_dir, + args.output_dir, + args.src_dirs, + args.strip_unused, + args.verbose, + ) else: try: Import("env") - print("Running i18n generation script from PlatformIO...") main() except NameError: pass