eric/Crosspoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Language updates

Browse Source
This commit is contained in:
jpirnay
2026-05-03 10:29:10 +02:00
parent 28326322c7
commit 6b002abc38
10 changed files with 733 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/gen_i18n.py
+120 -128
View File
@@ -206,36 +206,21 @@ def load_translations(
if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", key):
raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
# Build translations dict, filling missing keys from English.
# Non-English values that are absent or blank fall back to English at runtime.
# A literal empty string in a non-English YAML file is treated as an intentional
# empty translation and stored as a single space to distinguish it from fallback.
# Build translations dict, filling missing keys from English
inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
translations: Dict[str, List[str]] = {}
for key in string_keys:
row: List[str] = []
for lang_idx, fname in enumerate(ordered_files):
data = parsed[fname]
if key not in data:
value = ""
if fname != english_file:
inherited_sets[lang_idx].add(key)
if verbose:
print(
f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
)
else:
value = data[key]
if fname != english_file:
if value == "":
value = " "
elif not value.strip():
value = ""
inherited_sets[lang_idx].add(key)
if verbose:
print(
f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
)
value = data.get(key, "")
if not value.strip() and fname != english_file:
value = english_data[key]
inherited_sets[lang_idx].add(key)
if verbose:
print(
f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
)
row.append(value)
translations[key] = row
@@ -458,7 +443,11 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List
last_space = idx
# Handle escapes to step correctly
idx += 2 if current[idx] == "\\" else 1
if current[idx] == "\\":
idx += 2
else:
idx += 1
# If we found a space, split after it
if last_space != -1:
# Include the space in the first line
@@ -489,12 +478,8 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List
def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str:
"""Return a sorted string of every unique character used in a language."""
chars = set()
english_index = 0
for values in translations.values():
text = values[lang_index]
if lang_index != english_index and text == "":
text = values[english_index]
for ch in text:
for ch in values[lang_index]:
chars.add(ord(ch))
return "".join(chr(cp) for cp in sorted(chars))
@@ -524,19 +509,17 @@ def generate_keys_header(
]
for code in languages:
lines.extend(
(
f"extern const char STRINGS_{code}_DATA[];",
f"extern const uint16_t OFFSETS_{code}[];",
)
)
lines.append(f"extern const char STRINGS_{code}_DATA[];")
lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("} // namespace i18n_strings")
lines.append("")
# Language enum
lines.append("// Language enum")
lines.append("enum class Language : uint8_t {")
lines.extend(f" {lang} = {i}," for i, lang in enumerate(languages))
for i, lang in enumerate(languages):
lines.append(f" {lang} = {i},")
lines.append(" _COUNT")
lines.append("};")
lines.append("")
@@ -576,11 +559,9 @@ def generate_keys_header(
lines.append("inline LangStrings getLanguageStrings(Language lang) {")
lines.append(" switch (lang) {")
for code in languages:
lines.extend(
(
f" case Language::{code}:",
f" return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};",
)
lines.append(f" case Language::{code}:")
lines.append(
f" return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};"
)
first_code = languages[0]
lines.append(" default:")
@@ -608,10 +589,8 @@ def generate_keys_header(
)
sorted_indices = [english_idx] + rest
lines.append("// Sorted language indices by code (auto-generated by gen_i18n.py)")
lines.extend(
f"// {rank:>2}: {languages[idx]:<4} {language_names[idx]}"
for rank, idx in enumerate(sorted_indices)
)
for rank, idx in enumerate(sorted_indices):
lines.append(f"// {rank:>2}: {languages[idx]:<4} {language_names[idx]}")
lines.append(
"constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {"
f"{', '.join(str(i) for i in sorted_indices)}"
@@ -622,6 +601,23 @@ def generate_keys_header(
"static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount(),"
)
lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");')
lines.append("")
# V1 language.bin migration table -- frozen enum order from commit 2f969a9.
# Maps the old uint8_t index stored on disk to the current Language enum.
# If a Language enum value listed here is ever removed, this will fail to
# compile, signalling that the migration table needs updating.
v1_codes = [
"EN", "ES", "FR", "DE", "CS", "PT", "RU", "SV", "RO", "CA", "UK",
"BE", "IT", "PL", "FI", "DA", "NL", "TR", "KK", "HU", "LT", "SI",
]
lines.append("// V1 language.bin migration table (frozen enum order from 2f969a9)")
lines.append("constexpr Language V1_LANGUAGES[] = {")
lines.append(" " + ", ".join(f"Language::{c}" for c in v1_codes) + ",")
lines.append("};")
lines.append(
f"constexpr uint8_t V1_LANGUAGE_COUNT = {len(v1_codes)};"
)
_write_file(output_path, lines, verbose)
@@ -645,13 +641,11 @@ def generate_strings_header(
]
for code in languages:
lines.extend(
(
f"extern const char STRINGS_{code}_DATA[];",
f"extern const uint16_t OFFSETS_{code}[];",
)
)
lines.extend(("", "} // namespace i18n_strings"))
lines.append(f"extern const char STRINGS_{code}_DATA[];")
lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("")
lines.append("} // namespace i18n_strings")
_write_file(output_path, lines, verbose)
@@ -671,10 +665,11 @@ def generate_strings_cpp(
"",
"#include <cstddef>",
"",
"// Language codes",
"const char* const LANGUAGE_CODES[] = {",
]
# LANGUAGE_CODES array
lines.append("// Language codes")
lines.append("const char* const LANGUAGE_CODES[] = {")
for code in languages:
_append_string_entry(lines, code)
lines.append("};")
@@ -697,29 +692,56 @@ def generate_strings_cpp(
lines.append("};")
lines.append("")
# Per-language flat string blobs and offset tables
# Per-language flat string blobs and offset tables.
# Non-English languages skip strings identical to English; their offset
# tables use bit 15 (0x8000) to flag "use English blob at offset & 0x7FFF".
lines.append("namespace i18n_strings {")
lines.append("")
en_strings = [translations[key][0] for key in string_keys]
en_offsets: List[int] = []
for lang_idx, code in enumerate(languages):
lang_strings = [translations[key][lang_idx] for key in string_keys]
is_english = lang_idx == 0
# Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator)
offsets: List[int] = []
current_offset = 0
for s in lang_strings:
offsets.append(current_offset)
current_offset += len(s.encode("utf-8")) + 1
if current_offset > 65535:
raise ValueError(
f"Language {code}: total string data ({current_offset} bytes) "
"exceeds uint16_t offset range (65535)"
)
if is_english:
# Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator)
offsets: List[int] = []
current_offset = 0
for s in lang_strings:
offsets.append(current_offset)
current_offset += len(s.encode("utf-8")) + 1
if current_offset > 0x7FFF:
raise ValueError(
f"Language {code}: blob size ({current_offset} bytes) exceeds "
"15-bit offset limit (32767)"
)
en_offsets = list(offsets)
blob_strings = lang_strings
else:
offsets = []
current_offset = 0
blob_strings = []
for i, (s, en_s) in enumerate(zip(lang_strings, en_strings)):
if s == en_s:
offsets.append(en_offsets[i] | 0x8000)
else:
offsets.append(current_offset)
current_offset += len(s.encode("utf-8")) + 1
blob_strings.append(s)
if current_offset > 0x7FFF:
raise ValueError(
f"Language {code}: blob size ({current_offset} bytes) exceeds "
"15-bit offset limit (32767)"
)
# Flat string data blob — all strings concatenated with \0 separators.
lines.append(f"const char STRINGS_{code}_DATA[] =")
for text in lang_strings:
for text in blob_strings:
_append_string_data_entry(lines, text)
if not blob_strings:
_append_string_data_entry(lines, "")
lines.append(";")
lines.append("")
@@ -738,13 +760,13 @@ def generate_strings_cpp(
# Compile-time size checks
lines.append("// Compile-time validation of array sizes")
for code in languages:
lines.extend(
(
f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) / sizeof(i18n_strings::OFFSETS_{code}[0]) ==",
" static_cast<size_t>(StrId::_COUNT),",
f' "OFFSETS_{code} size mismatch");',
)
lines.append(
f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) "
f"/ sizeof(i18n_strings::OFFSETS_{code}[0]) =="
)
lines.append(" static_cast<size_t>(StrId::_COUNT),")
lines.append(f' "OFFSETS_{code} size mismatch");')
_write_file(output_path, lines, verbose)
@@ -760,43 +782,20 @@ def _print_language_table(
string_keys: List[str],
unused_keys: Set[str],
data_sizes: List[int],
translations: Dict[str, List[str]],
) -> None:
"""Print a per-language summary table."""
total = len(string_keys)
headers = (
"Language",
"Code",
"Own",
"Fallback",
"Unused",
"Data (B)",
"Unique (B)",
)
headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)")
rows = []
for lang_idx, (code, name, inherited, size) in enumerate(
zip(language_codes, language_names, inherited_sets, data_sizes)
for code, name, inherited, size in zip(
language_codes, language_names, inherited_sets, data_sizes
):
own = total - len(inherited)
fallback = len(inherited)
# strings this language translated but the code never calls
unused = len(unused_keys - inherited)
unique_size = sum(
len(s.encode("utf-8")) + 1
for s in {translations[k][lang_idx] for k in string_keys}
)
rows.append(
(
name,
code,
str(own),
str(fallback),
str(unused),
str(size),
str(unique_size),
)
)
rows.append((name, code, str(own), str(fallback), str(unused), str(size)))
# EN first, then alphabetically by ISO code
rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
@@ -821,34 +820,18 @@ def _print_language_table(
for row in rows:
_safe_print(fmt.format(*row))
used = total - len(unused_keys)
total_size = sum(data_sizes)
dedup_size = sum(data_sizes)
n_lang = len(rows)
n_keys = len(string_keys)
# Current layout: uint16_t offset table (2 B per string per language)
offset_table_size = n_lang * n_keys * 2
current_total = total_size + offset_table_size
# Estimate the original pointer-based layout using deduplicated string storage.
unique_strings: Set[str] = set()
for values in translations.values():
unique_strings.update(values)
unique_string_size = sum(len(s.encode("utf-8")) + 1 for s in unique_strings)
old_pointer_table_size = n_lang * n_keys * 4
old_total = unique_string_size + old_pointer_table_size
saved = old_total - current_total
current_total = dedup_size + offset_table_size
print(
f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}"
)
print(
f" Flash (now): {total_size:>7,} B strings + {offset_table_size:>6,} B offset tables (uint16_t)"
f" Flash: {dedup_size:>7,} B strings (deduped) + {offset_table_size:>6,} B offset tables"
f" = {current_total:>7,} B"
)
print(
f" Flash (pointer model, deduped): {unique_string_size:>7,} B unique strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)"
f" = {old_total:>7,} B"
)
print(f" Estimated savings vs pointer model: {saved:,} B")
def _append_string_data_entry(lines: List[str], text: str) -> None:
@@ -951,12 +934,22 @@ def main(
print()
sys.exit(1)
# Compute per-language data blob sizes:
# sum of UTF-8 byte length + 1 (null terminator) per string
data_sizes = [
sum(len(translations[k][i].encode("utf-8")) + 1 for k in string_keys)
for i in range(len(languages))
]
# Compute per-language data blob sizes (after dedup).
# Non-English languages omit strings identical to English.
data_sizes = []
for i in range(len(languages)):
if i == 0:
data_sizes.append(
sum(len(translations[k][0].encode("utf-8")) + 1 for k in string_keys)
)
else:
data_sizes.append(
sum(
len(translations[k][i].encode("utf-8")) + 1
for k in string_keys
if translations[k][i] != translations[k][0]
)
)
_print_language_table(
languages,
@@ -965,7 +958,6 @@ def main(
string_keys,
unused_set,
data_sizes,
translations,
)
print()
@@ -1062,4 +1054,4 @@ else:
Import("env")
main(strip_unused=True)
except NameError:
pass
pass