eric/Crosspoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage scripts

Browse Source
This commit is contained in:
jpirnay
2026-03-30 18:12:52 +02:00
parent 474dfe5490
commit c8b166606e
4 changed files with 9459 additions and 5909 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/I18n/I18nKeys.h
+143 -47
View File
@@ -3,27 +3,48 @@
// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.
// Forward declaration for string arrays
// Forward declarations for flat string data blobs and offset tables
namespace i18n_strings {
extern const char* const STRINGS_EN[];
extern const char* const STRINGS_ES[];
extern const char* const STRINGS_FR[];
extern const char* const STRINGS_DE[];
extern const char* const STRINGS_CS[];
extern const char* const STRINGS_PO[];
extern const char* const STRINGS_RU[];
extern const char* const STRINGS_SV[];
extern const char* const STRINGS_RO[];
extern const char* const STRINGS_CA[];
extern const char* const STRINGS_UK[];
extern const char* const STRINGS_BE[];
extern const char* const STRINGS_IT[];
extern const char* const STRINGS_PL[];
extern const char* const STRINGS_FI[];
extern const char* const STRINGS_DA[];
extern const char* const STRINGS_NL[];
extern const char* const STRINGS_TR[];
extern const char* const STRINGS_KK[];
extern const char STRINGS_EN_DATA[];
extern const uint16_t OFFSETS_EN[];
extern const char STRINGS_ES_DATA[];
extern const uint16_t OFFSETS_ES[];
extern const char STRINGS_FR_DATA[];
extern const uint16_t OFFSETS_FR[];
extern const char STRINGS_DE_DATA[];
extern const uint16_t OFFSETS_DE[];
extern const char STRINGS_CS_DATA[];
extern const uint16_t OFFSETS_CS[];
extern const char STRINGS_PO_DATA[];
extern const uint16_t OFFSETS_PO[];
extern const char STRINGS_RU_DATA[];
extern const uint16_t OFFSETS_RU[];
extern const char STRINGS_SV_DATA[];
extern const uint16_t OFFSETS_SV[];
extern const char STRINGS_RO_DATA[];
extern const uint16_t OFFSETS_RO[];
extern const char STRINGS_CA_DATA[];
extern const uint16_t OFFSETS_CA[];
extern const char STRINGS_UK_DATA[];
extern const uint16_t OFFSETS_UK[];
extern const char STRINGS_BE_DATA[];
extern const uint16_t OFFSETS_BE[];
extern const char STRINGS_IT_DATA[];
extern const uint16_t OFFSETS_IT[];
extern const char STRINGS_PL_DATA[];
extern const uint16_t OFFSETS_PL[];
extern const char STRINGS_FI_DATA[];
extern const uint16_t OFFSETS_FI[];
extern const char STRINGS_DA_DATA[];
extern const uint16_t OFFSETS_DA[];
extern const char STRINGS_NL_DATA[];
extern const uint16_t OFFSETS_NL[];
extern const char STRINGS_TR_DATA[];
extern const uint16_t OFFSETS_TR[];
extern const char STRINGS_KK_DATA[];
extern const uint16_t OFFSETS_KK[];
extern const char STRINGS_PT_DATA[];
extern const uint16_t OFFSETS_PT[];
} // namespace i18n_strings
// Language enum
@@ -33,7 +54,7 @@ enum class Language : uint8_t {
FR = 2,
DE = 3,
CS = 4,
PT = 5,
PO = 5,
RU = 6,
SV = 7,
RO = 8,
@@ -47,6 +68,7 @@ enum class Language : uint8_t {
NL = 16,
TR = 17,
KK = 18,
PT = 19,
_COUNT
};
@@ -126,6 +148,7 @@ enum class StrId : uint16_t {
STR_IMAGES_DISPLAY,
STR_IMAGES_PLACEHOLDER,
STR_IMAGES_SUPPRESS,
STR_CREATE_FALLBACK_FOR_INVALID_TOC,
STR_SHORT_PWR_BTN,
STR_ORIENTATION,
STR_SIDE_BTN_LAYOUT,
@@ -138,6 +161,43 @@ enum class StrId : uint16_t {
STR_HYPHENATION,
STR_TIME_TO_SLEEP,
STR_SHOW_HIDDEN_FILES,
STR_USE_CLOCK,
STR_CLOCK_SETTINGS,
STR_CLOCK_SETTINGS_WARNING,
STR_CLOCK,
STR_CLOCK_FORMAT,
STR_TIMEZONE,
STR_24H,
STR_12H,
STR_TZ_UTC,
STR_TZ_CET,
STR_TZ_EET,
STR_TZ_EST,
STR_TZ_CST,
STR_TZ_MST,
STR_TZ_PST,
STR_TZ_AEST,
STR_TZ_NZST,
STR_TZ_MSK,
STR_TZ_UTC_MINUS3,
STR_TZ_UTC_PLUS4,
STR_TZ_IST,
STR_TZ_UTC_PLUS7,
STR_TZ_UTC_PLUS8,
STR_TZ_UTC_PLUS9,
STR_SYNC_TIME,
STR_DETECT_TIMEZONE,
STR_SYNCING_CLOCK,
STR_DETECTING_TIMEZONE,
STR_TIME_SYNCED,
STR_TIME_SYNC_FAILED,
STR_CLOCK_DRIFT,
STR_LAST_NTP_SYNC,
STR_TIMEZONE_DETECTED,
STR_TIMEZONE_DETECT_FAILED,
STR_DST_ACTIVE,
STR_DST_INACTIVE,
STR_DST_UNKNOWN,
STR_REFRESH_FREQ,
STR_KOREADER_SYNC,
STR_CHECK_UPDATES,
@@ -159,6 +219,11 @@ enum class StrId : uint16_t {
STR_KOREADER_AUTH,
STR_SYNC_READY,
STR_AUTH_FAILED,
STR_REGISTER,
STR_REGISTERING,
STR_REGISTER_SUCCESS,
STR_REGISTER_FAILED,
STR_USERNAME_TAKEN,
STR_DONE,
STR_CLEAR_CACHE_WARNING_1,
STR_CLEAR_CACHE_WARNING_2,
@@ -286,6 +351,8 @@ enum class StrId : uint16_t {
STR_REMAP_FRONT_BUTTONS,
STR_OPDS_BROWSER,
STR_COVER_CUSTOM,
STR_PAGE_OVERLAY,
STR_RECENTS,
STR_MENU_RECENT_BOOKS,
STR_NO_RECENT_BOOKS,
STR_CALIBRE_DESC,
@@ -310,6 +377,7 @@ enum class StrId : uint16_t {
STR_SYNC_PROGRESS,
STR_DELETE_CACHE,
STR_DELETE,
STR_REMOVE,
STR_DISPLAY_QR,
STR_CHAPTER_PREFIX,
STR_PAGES_SEPARATOR,
@@ -320,6 +388,8 @@ enum class StrId : uint16_t {
STR_CALC_HASH,
STR_HASH_FAILED,
STR_FETCH_PROGRESS,
STR_MAPPING_REMOTE,
STR_MAPPING_LOCAL,
STR_UPLOAD_PROGRESS,
STR_NO_CREDENTIALS_MSG,
STR_KOREADER_SETUP_HINT,
@@ -346,6 +416,10 @@ enum class StrId : uint16_t {
STR_SCREENSHOT_BUTTON,
STR_AUTO_TURN_ENABLED,
STR_AUTO_TURN_PAGES_PER_MIN,
STR_INFO,
STR_AUTHOR,
STR_SERIES,
STR_FILE_SIZE,
STR_SLEEP_COVER_OVERLAY,
STR_OVERLAY_WHITE,
STR_OVERLAY_GRAY,
@@ -354,53 +428,76 @@ enum class StrId : uint16_t {
STR_OVERLAY_READING_PROGRESS,
STR_OVERLAY_READING_PROGRESS_NO_TOTAL,
STR_OVERLAY_CHAPTER_PAGE_SUFFIX,
STR_SYSTEM_INFO,
STR_LOAD_XTC_FAILED,
STR_LOAD_EPUB_FAILED,
STR_FW_VERSION,
STR_CHIP,
STR_CPU,
STR_MHZ,
STR_FREE_RAM,
STR_MIN_FREE,
STR_MAX_BLOCK,
STR_FLASH_USED,
STR_UPTIME,
STR_CHARGING,
STR_GATHERING_DATA,
STR_READING,
// Sentinel - must be last
_COUNT
};
// Helper function to get string array for a language
inline const char* const* getStringArray(Language lang) {
// Holds a flat string blob and its offset table for one language
struct LangStrings {
const char* data;
const uint16_t* offsets;
};
// Helper function to get string data for a language
inline LangStrings getLanguageStrings(Language lang) {
switch (lang) {
case Language::EN:
return i18n_strings::STRINGS_EN;
return {i18n_strings::STRINGS_EN_DATA, i18n_strings::OFFSETS_EN};
case Language::ES:
return i18n_strings::STRINGS_ES;
return {i18n_strings::STRINGS_ES_DATA, i18n_strings::OFFSETS_ES};
case Language::FR:
return i18n_strings::STRINGS_FR;
return {i18n_strings::STRINGS_FR_DATA, i18n_strings::OFFSETS_FR};
case Language::DE:
return i18n_strings::STRINGS_DE;
return {i18n_strings::STRINGS_DE_DATA, i18n_strings::OFFSETS_DE};
case Language::CS:
return i18n_strings::STRINGS_CS;
case Language::PT:
return i18n_strings::STRINGS_PO;
return {i18n_strings::STRINGS_CS_DATA, i18n_strings::OFFSETS_CS};
case Language::PO:
return {i18n_strings::STRINGS_PO_DATA, i18n_strings::OFFSETS_PO};
case Language::RU:
return i18n_strings::STRINGS_RU;
return {i18n_strings::STRINGS_RU_DATA, i18n_strings::OFFSETS_RU};
case Language::SV:
return i18n_strings::STRINGS_SV;
return {i18n_strings::STRINGS_SV_DATA, i18n_strings::OFFSETS_SV};
case Language::RO:
return i18n_strings::STRINGS_RO;
return {i18n_strings::STRINGS_RO_DATA, i18n_strings::OFFSETS_RO};
case Language::CA:
return i18n_strings::STRINGS_CA;
return {i18n_strings::STRINGS_CA_DATA, i18n_strings::OFFSETS_CA};
case Language::UK:
return i18n_strings::STRINGS_UK;
return {i18n_strings::STRINGS_UK_DATA, i18n_strings::OFFSETS_UK};
case Language::BE:
return i18n_strings::STRINGS_BE;
return {i18n_strings::STRINGS_BE_DATA, i18n_strings::OFFSETS_BE};
case Language::IT:
return i18n_strings::STRINGS_IT;
return {i18n_strings::STRINGS_IT_DATA, i18n_strings::OFFSETS_IT};
case Language::PL:
return i18n_strings::STRINGS_PL;
return {i18n_strings::STRINGS_PL_DATA, i18n_strings::OFFSETS_PL};
case Language::FI:
return i18n_strings::STRINGS_FI;
return {i18n_strings::STRINGS_FI_DATA, i18n_strings::OFFSETS_FI};
case Language::DA:
return i18n_strings::STRINGS_DA;
return {i18n_strings::STRINGS_DA_DATA, i18n_strings::OFFSETS_DA};
case Language::NL:
return i18n_strings::STRINGS_NL;
return {i18n_strings::STRINGS_NL_DATA, i18n_strings::OFFSETS_NL};
case Language::TR:
return i18n_strings::STRINGS_TR;
return {i18n_strings::STRINGS_TR_DATA, i18n_strings::OFFSETS_TR};
case Language::KK:
return i18n_strings::STRINGS_KK;
return {i18n_strings::STRINGS_KK_DATA, i18n_strings::OFFSETS_KK};
case Language::PT:
return {i18n_strings::STRINGS_PT_DATA, i18n_strings::OFFSETS_PT};
default:
return i18n_strings::STRINGS_EN;
return {i18n_strings::STRINGS_EN_DATA, i18n_strings::OFFSETS_EN};
}
}
@@ -408,9 +505,8 @@ inline const char* const* getStringArray(Language lang) {
constexpr uint8_t getLanguageCount() { return static_cast<uint8_t>(Language::_COUNT); }
// Sorted language indices by code (auto-generated by gen_i18n.py)
// Order: English, Беларуская, Català, Čeština, Dansk, Deutsch, Español, Suomi, Français, Italiano
// Қазақша, Nederlands, Polski, Português (Brasil), Română, Русский, Svenska, Türkçe, Українська
constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {0, 11, 9, 4, 15, 3, 1, 14, 2, 12, 18, 16, 13, 5, 8, 6, 7, 17, 10};
// Order: English, Беларуская, Català, Čeština, Dansk, Deutsch, Español, Suomi, Français, Italiano, Қазақша, Nederlands, Polski, Português (Brasil), Português (Portugal), Română, Русский, Svenska, Türkçe, Українська
constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {0, 11, 9, 4, 15, 3, 1, 14, 2, 12, 18, 16, 13, 5, 19, 8, 6, 7, 17, 10};
static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount(),
"SORTED_LANGUAGE_INDICES size mismatch");
lib/I18n/I18nStrings.cpp
+8893 -5727
View File
File diff suppressed because it is too large Load Diff
lib/I18n/I18nStrings.h
+40 -21
View File
@@ -1,30 +1,49 @@
#pragma once
#include <string>
#include "I18nKeys.h"
// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.
namespace i18n_strings {
extern const char* const STRINGS_EN[];
extern const char* const STRINGS_ES[];
extern const char* const STRINGS_FR[];
extern const char* const STRINGS_DE[];
extern const char* const STRINGS_CS[];
extern const char* const STRINGS_PO[];
extern const char* const STRINGS_RU[];
extern const char* const STRINGS_SV[];
extern const char* const STRINGS_RO[];
extern const char* const STRINGS_CA[];
extern const char* const STRINGS_UK[];
extern const char* const STRINGS_BE[];
extern const char* const STRINGS_IT[];
extern const char* const STRINGS_PL[];
extern const char* const STRINGS_FI[];
extern const char* const STRINGS_DA[];
extern const char* const STRINGS_NL[];
extern const char* const STRINGS_TR[];
extern const char* const STRINGS_KK[];
extern const char STRINGS_EN_DATA[];
extern const uint16_t OFFSETS_EN[];
extern const char STRINGS_ES_DATA[];
extern const uint16_t OFFSETS_ES[];
extern const char STRINGS_FR_DATA[];
extern const uint16_t OFFSETS_FR[];
extern const char STRINGS_DE_DATA[];
extern const uint16_t OFFSETS_DE[];
extern const char STRINGS_CS_DATA[];
extern const uint16_t OFFSETS_CS[];
extern const char STRINGS_PO_DATA[];
extern const uint16_t OFFSETS_PO[];
extern const char STRINGS_RU_DATA[];
extern const uint16_t OFFSETS_RU[];
extern const char STRINGS_SV_DATA[];
extern const uint16_t OFFSETS_SV[];
extern const char STRINGS_RO_DATA[];
extern const uint16_t OFFSETS_RO[];
extern const char STRINGS_CA_DATA[];
extern const uint16_t OFFSETS_CA[];
extern const char STRINGS_UK_DATA[];
extern const uint16_t OFFSETS_UK[];
extern const char STRINGS_BE_DATA[];
extern const uint16_t OFFSETS_BE[];
extern const char STRINGS_IT_DATA[];
extern const uint16_t OFFSETS_IT[];
extern const char STRINGS_PL_DATA[];
extern const uint16_t OFFSETS_PL[];
extern const char STRINGS_FI_DATA[];
extern const uint16_t OFFSETS_FI[];
extern const char STRINGS_DA_DATA[];
extern const uint16_t OFFSETS_DA[];
extern const char STRINGS_NL_DATA[];
extern const uint16_t OFFSETS_NL[];
extern const char STRINGS_TR_DATA[];
extern const uint16_t OFFSETS_TR[];
extern const char STRINGS_KK_DATA[];
extern const uint16_t OFFSETS_KK[];
extern const char STRINGS_PT_DATA[];
extern const uint16_t OFFSETS_PT[];
} // namespace i18n_strings
scripts/gen_i18n.py
+383 -114
View File
@@ -15,18 +15,25 @@ Each YAML file must contain:
The English file is the reference. Missing keys in other languages are
automatically filled from English, with a warning.
Usage:
python gen_i18n.py <translations_dir> <output_dir>
By default the script scans the src/ and lib/ trees for STR_* references and
reports any translation keys that are never used. Pass --strip-unused to
omit those keys from the generated output entirely.
Example:
Usage:
python gen_i18n.py [translations_dir [output_dir]] [options]
Examples:
python gen_i18n.py
python gen_i18n.py lib/I18n/translations lib/I18n/
python gen_i18n.py --strip-unused
python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont
"""
import sys
import os
import re
from pathlib import Path
from typing import List, Dict, Tuple
from typing import Dict, List, Optional, Set, Tuple
# ---------------------------------------------------------------------------
@@ -105,7 +112,8 @@ def parse_yaml_file(filepath: str) -> Dict[str, str]:
def load_translations(
translations_dir: str,
) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]]]:
verbose: bool = False,
) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]:
"""
Read every YAML file in *translations_dir* and return:
language_codes e.g. ["EN", "ES", ...]
@@ -138,6 +146,31 @@ def load_translations(
if english_file is None:
raise ValueError("No YAML file with _language_code: EN found")
duplicate_orders: Dict[str, List[str]] = {}
order_to_files: Dict[str, List[str]] = {}
for fname, data in parsed.items():
order = data.get("_order")
if not order:
continue
order_to_files.setdefault(order, []).append(fname)
for order, files in order_to_files.items():
if len(files) > 1:
duplicate_orders[order] = sorted(files)
if duplicate_orders:
duplicate_messages = [
f"_order {order}: {', '.join(files)}"
for order, files in sorted(
duplicate_orders.items(), key=lambda item: int(item[0])
)
]
raise ValueError(
"Duplicate _order values found:\n "
+ "\n ".join(duplicate_messages)
+ "\nEach _order value must be unique to ensure a deterministic language order."
)
# Order: English first, then by _order metadata (falls back to filename)
def sort_key(fname: str) -> Tuple[int, int, str]:
"""English always first (0), then by _order, then by filename."""
@@ -174,16 +207,20 @@ def load_translations(
raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
# Build translations dict, filling missing keys from English
inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
translations: Dict[str, List[str]] = {}
for key in string_keys:
row: List[str] = []
for fname in ordered_files:
for lang_idx, fname in enumerate(ordered_files):
data = parsed[fname]
value = data.get(key, "")
if not value.strip() and fname != english_file:
value = english_data[key]
lang_code = parsed[fname].get("_language_code", fname)
print(f" INFO: '{key}' missing in {lang_code}, using English fallback")
inherited_sets[lang_idx].add(key)
if verbose:
print(
f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
)
row.append(value)
translations[key] = row
@@ -195,59 +232,72 @@ def load_translations(
extra = [k for k in data if not k.startswith("_") and k not in english_data]
if extra:
lang_code = data.get("_language_code", fname)
print(f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}")
if verbose:
print(
f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}"
)
print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys")
return language_codes, language_names, string_keys, translations
if verbose:
print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys")
return language_codes, language_names, string_keys, translations, inherited_sets
# ---------------------------------------------------------------------------
# Unused-string detection
# ---------------------------------------------------------------------------
_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"}
def find_used_string_keys(
src_dirs: List[str],
skip_filenames: Optional[Set[str]] = None,
) -> Set[str]:
"""
Scan C/C++ source files under *src_dirs* for STR_* identifiers.
Files whose basename appears in *skip_filenames* are skipped so that
the generated I18n files don't count as "usage" of themselves.
Returns the set of all STR_KEY names that appear at least once.
"""
if skip_filenames is None:
skip_filenames = _GENERATED_FILENAMES
pattern = re.compile(r"\bSTR_[A-Za-z0-9_]+\b")
used: Set[str] = set()
for src_dir in src_dirs:
p = Path(src_dir)
if not p.is_dir():
continue
for f in p.rglob("*"):
if f.suffix not in {".cpp", ".h", ".c"}:
continue
if f.name in skip_filenames:
continue
try:
text = f.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
for m in pattern.finditer(text):
used.add(m.group(0))
return used
def report_unused_keys(
string_keys: List[str],
used_keys: Set[str],
) -> List[str]:
"""Return a sorted list of keys from *string_keys* absent in *used_keys*."""
return [k for k in sorted(string_keys) if k not in used_keys]
# ---------------------------------------------------------------------------
# C++ string escaping
# ---------------------------------------------------------------------------
LANG_ABBREVIATIONS = {
"english": "EN",
"español": "ES",
"espanol": "ES",
"italiano": "IT",
"svenska": "SV",
"français": "FR",
"francais": "FR",
"deutsch": "DE",
"german": "DE",
"polski": "PL",
"português": "PT",
"portugues": "PT",
"português (brasil)": "PO",
"中文": "ZH",
"chinese": "ZH",
"日本語": "JA",
"japanese": "JA",
"한국어": "KO",
"korean": "KO",
"русский": "RU",
"russian": "RU",
"العربية": "AR",
"arabic": "AR",
"עברית": "HE",
"hebrew": "HE",
"فارسی": "FA",
"persian": "FA",
"čeština": "CS",
"türkçe": "TR",
"turkish": "TR",
"Қазақша": "KK",
"kazakh": "KK",
}
def get_lang_abbreviation(lang_code: str, lang_name: str) -> str:
"""Return a 2-letter abbreviation for a language."""
lower = lang_name.lower()
if lower in LANG_ABBREVIATIONS:
return LANG_ABBREVIATIONS[lower]
return lang_code[:2].upper()
def escape_cpp_string(s: str) -> List[str]:
r"""
@@ -390,6 +440,7 @@ def generate_keys_header(
language_names: List[str],
string_keys: List[str],
output_path: str,
verbose: bool = False,
) -> None:
"""Generate I18nKeys.h."""
lines: List[str] = [
@@ -398,13 +449,13 @@ def generate_keys_header(
"",
"// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
"",
"// Forward declaration for string arrays",
"// Forward declarations for flat string data blobs and offset tables",
"namespace i18n_strings {",
]
for code, name in zip(languages, language_names):
abbrev = get_lang_abbreviation(code, name)
lines.append(f"extern const char* const STRINGS_{abbrev}[];")
for code in languages:
lines.append(f"extern const char STRINGS_{code}_DATA[];")
lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("} // namespace i18n_strings")
lines.append("")
@@ -436,17 +487,28 @@ def generate_keys_header(
lines.append("};")
lines.append("")
# getStringArray helper
lines.append("// Helper function to get string array for a language")
lines.append("inline const char* const* getStringArray(Language lang) {")
# LangStrings struct
lines.append("// Holds a flat string blob and its offset table for one language")
lines.append("struct LangStrings {")
lines.append(" const char* data;")
lines.append(" const uint16_t* offsets;")
lines.append("};")
lines.append("")
# getLanguageStrings helper
lines.append("// Helper function to get string data for a language")
lines.append("inline LangStrings getLanguageStrings(Language lang) {")
lines.append(" switch (lang) {")
for code, name in zip(languages, language_names):
abbrev = get_lang_abbreviation(code, name)
for code in languages:
lines.append(f" case Language::{code}:")
lines.append(f" return i18n_strings::STRINGS_{abbrev};")
first_abbrev = get_lang_abbreviation(languages[0], language_names[0])
lines.append(
f" return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};"
)
first_code = languages[0]
lines.append(" default:")
lines.append(f" return i18n_strings::STRINGS_{first_abbrev};")
lines.append(
f" return {{i18n_strings::STRINGS_{first_code}_DATA, i18n_strings::OFFSETS_{first_code}}};"
)
lines.append(" }")
lines.append("}")
lines.append("")
@@ -467,21 +529,9 @@ def generate_keys_header(
key=lambda i: languages[i],
)
sorted_indices = [english_idx] + rest
names = [language_names[i] for i in sorted_indices]
comment_names = ", ".join(names)
comment_names = ", ".join(language_names[i] for i in sorted_indices)
lines.append("// Sorted language indices by code (auto-generated by gen_i18n.py)")
if len(f"// Order: {comment_names}") <= 120:
lines.append(f"// Order: {comment_names}")
else:
current = "// Order: "
for name in names:
candidate = current + name + ", "
if len(candidate) > 100:
lines.append(current.rstrip(", "))
current = "// " + name + ", "
else:
current = candidate
lines.append(current.rstrip(", "))
lines.append(f"// Order: {comment_names}")
lines.append(
"constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {"
f"{', '.join(str(i) for i in sorted_indices)}"
@@ -493,19 +543,18 @@ def generate_keys_header(
)
lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");')
_write_file(output_path, lines)
_write_file(output_path, lines, verbose)
def generate_strings_header(
languages: List[str],
language_names: List[str],
output_path: str,
verbose: bool = False,
) -> None:
"""Generate I18nStrings.h."""
lines: List[str] = [
"#pragma once",
"#include <string>",
"",
'#include "I18nKeys.h"',
"",
"// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
@@ -514,13 +563,13 @@ def generate_strings_header(
"",
]
for code, name in zip(languages, language_names):
abbrev = get_lang_abbreviation(code, name)
lines.append(f"extern const char* const STRINGS_{abbrev}[];")
for code in languages:
lines.append(f"extern const char STRINGS_{code}_DATA[];")
lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("")
lines.append("} // namespace i18n_strings")
_write_file(output_path, lines)
_write_file(output_path, lines, verbose)
def generate_strings_cpp(
@@ -529,10 +578,12 @@ def generate_strings_cpp(
string_keys: List[str],
translations: Dict[str, List[str]],
output_path: str,
verbose: bool = False,
) -> None:
"""Generate I18nStrings.cpp."""
lines: List[str] = [
'#include "I18nStrings.h"',
"#include <cstddef>",
"",
"// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
"",
@@ -555,18 +606,41 @@ def generate_strings_cpp(
lines.append("};")
lines.append("")
# Per-language string arrays
# Per-language flat string blobs and offset tables
lines.append("namespace i18n_strings {")
lines.append("")
for lang_idx, (code, name) in enumerate(zip(languages, language_names)):
abbrev = get_lang_abbreviation(code, name)
lines.append(f"const char* const STRINGS_{abbrev}[] = {{")
for lang_idx, code in enumerate(languages):
lang_strings = [translations[key][lang_idx] for key in string_keys]
for key in string_keys:
text = translations[key][lang_idx]
_append_string_entry(lines, text)
# Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator)
offsets: List[int] = []
current_offset = 0
for s in lang_strings:
offsets.append(current_offset)
current_offset += len(s.encode("utf-8")) + 1
if current_offset > 65535:
raise ValueError(
f"Language {code}: total string data ({current_offset} bytes) "
"exceeds uint16_t offset range (65535)"
)
# Flat string data blob — all strings concatenated with \0 separators.
# clang-format off/on avoids reformatting of the adjacent string literals.
lines.append("// clang-format off")
lines.append(f"const char STRINGS_{code}_DATA[] =")
for text in lang_strings:
_append_string_data_entry(lines, text)
lines.append(";")
lines.append("// clang-format on")
lines.append("")
# Offset table — one uint16_t per StrId
lines.append(f"const uint16_t OFFSETS_{code}[] = {{")
chunk_size = 12
for i in range(0, len(offsets), chunk_size):
chunk = offsets[i : i + chunk_size]
lines.append(" " + ", ".join(str(o) for o in chunk) + ",")
lines.append("};")
lines.append("")
@@ -575,16 +649,15 @@ def generate_strings_cpp(
# Compile-time size checks
lines.append("// Compile-time validation of array sizes")
for code, name in zip(languages, language_names):
abbrev = get_lang_abbreviation(code, name)
for code in languages:
lines.append(
f"static_assert(sizeof(i18n_strings::STRINGS_{abbrev}) "
f"/ sizeof(i18n_strings::STRINGS_{abbrev}[0]) =="
f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) "
f"/ sizeof(i18n_strings::OFFSETS_{code}[0]) =="
)
lines.append(" static_cast<size_t>(StrId::_COUNT),")
lines.append(f' "STRINGS_{abbrev} size mismatch");')
lines.append(f' "OFFSETS_{code} size mismatch");')
_write_file(output_path, lines)
_write_file(output_path, lines, verbose)
# ---------------------------------------------------------------------------
@@ -592,6 +665,91 @@ def generate_strings_cpp(
# ---------------------------------------------------------------------------
def _print_language_table(
language_codes: List[str],
language_names: List[str],
inherited_sets: List[Set[str]],
string_keys: List[str],
unused_keys: Set[str],
data_sizes: List[int],
) -> None:
"""Print a per-language summary table."""
total = len(string_keys)
headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)")
rows = []
for code, name, inherited, size in zip(
language_codes, language_names, inherited_sets, data_sizes
):
own = total - len(inherited)
fallback = len(inherited)
# strings this language translated but the code never calls
unused = len(unused_keys - inherited)
rows.append((name, code, str(own), str(fallback), str(unused), str(size)))
# EN first, then alphabetically by ISO code
rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
col_widths = [len(h) for h in headers]
for row in rows:
for i, cell in enumerate(row):
col_widths[i] = max(col_widths[i], len(cell))
fmt = " ".join(f"{{:<{w}}}" for w in col_widths)
sep = " ".join("-" * w for w in col_widths)
def _safe_print(line: str) -> None:
print(
line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode(
sys.stdout.encoding or "utf-8", errors="replace"
)
)
_safe_print(fmt.format(*headers))
_safe_print(sep)
for row in rows:
_safe_print(fmt.format(*row))
used = total - len(unused_keys)
total_size = sum(data_sizes)
n_lang = len(rows)
n_keys = len(string_keys)
# Current layout: uint16_t offset table (2 B per string per language)
offset_table_size = n_lang * n_keys * 2
current_total = total_size + offset_table_size
# Previous layout: const char* pointer array (4 B per string per language)
old_pointer_table_size = n_lang * n_keys * 4
old_total = total_size + old_pointer_table_size
saved = old_total - current_total
print(
f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}"
)
print(
f" Flash (now): {total_size:>7,} B strings + {offset_table_size:>6,} B offset tables (uint16_t)"
f" = {current_total:>7,} B"
)
print(
f" Flash (before): {total_size:>7,} B strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)"
f" = {old_total:>7,} B"
)
print(f" Saved by offset tables: {saved:,} B")
def _append_string_data_entry(lines: List[str], text: str) -> None:
"""
Escape *text*, append a \\0 null separator, and format as indented C++
string literal lines for inclusion in a flat char data array blob.
"""
segments = escape_cpp_string(text)
# Append the null entry separator to the last segment
if segments and segments[-1] != "":
segments[-1] += "\\0"
elif segments:
segments[-1] = "\\0"
else:
segments = ["\\0"]
lines.extend(format_cpp_string_literal(segments))
def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None:
"""Escape *text*, format as indented C++ lines, append comma (and optional comment)."""
segments = escape_cpp_string(text)
@@ -601,11 +759,12 @@ def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None
lines.extend(formatted)
def _write_file(path: str, lines: List[str]) -> None:
def _write_file(path: str, lines: List[str], verbose: bool = False) -> None:
with open(path, "w", encoding="utf-8", newline="\n") as f:
f.write("\n".join(lines))
f.write("\n")
print(f"Generated: {path}")
if verbose:
print(f"Generated: {path}")
# ---------------------------------------------------------------------------
@@ -613,10 +772,17 @@ def _write_file(path: str, lines: List[str]) -> None:
# ---------------------------------------------------------------------------
def main(translations_dir=None, output_dir=None) -> None:
def main(
translations_dir: Optional[str] = None,
output_dir: Optional[str] = None,
src_dirs: Optional[List[str]] = None,
strip_unused: bool = False,
verbose: bool = False,
) -> None:
# Default paths (relative to project root)
default_translations_dir = "lib/I18n/translations"
default_output_dir = "lib/I18n/"
default_src_dirs = ["src", "lib"]
if translations_dir is None or output_dir is None:
if len(sys.argv) == 3:
@@ -627,6 +793,9 @@ def main(translations_dir=None, output_dir=None) -> None:
translations_dir = default_translations_dir
output_dir = default_output_dir
if src_dirs is None:
src_dirs = default_src_dirs
if not os.path.isdir(translations_dir):
print(f"Error: Translations directory not found: {translations_dir}")
sys.exit(1)
@@ -635,32 +804,91 @@ def main(translations_dir=None, output_dir=None) -> None:
print(f"Error: Output directory not found: {output_dir}")
sys.exit(1)
print(f"Reading translations from: {translations_dir}")
print(f"Output directory: {output_dir}")
print()
if verbose:
print(f"Reading translations from: {translations_dir}")
print(f"Output directory: {output_dir}")
print()
try:
languages, language_names, string_keys, translations = load_translations(
translations_dir
languages, language_names, string_keys, translations, inherited_sets = (
load_translations(translations_dir, verbose)
)
# --- Unused-string detection ---
scan_dirs = [d for d in src_dirs if os.path.isdir(d)]
if scan_dirs:
used_keys = find_used_string_keys(scan_dirs)
unused_set = set(report_unused_keys(string_keys, used_keys))
else:
used_keys = set(string_keys)
unused_set = set()
# --- Missing-string detection (used in code but absent from English) ---
missing_keys = sorted(used_keys - set(string_keys))
if missing_keys:
print(
f"\n CRITICAL: {len(missing_keys)} string(s) used in source but missing from english.yaml:"
)
for key in missing_keys:
print(f" - {key}")
print()
sys.exit(1)
# Compute per-language data blob sizes:
# sum of UTF-8 byte length + 1 (null terminator) per string
data_sizes = [
sum(len(translations[k][i].encode("utf-8")) + 1 for k in string_keys)
for i in range(len(languages))
]
_print_language_table(
languages,
language_names,
inherited_sets,
string_keys,
unused_set,
data_sizes,
)
print()
if verbose and unused_set:
print(f" Unused keys ({len(unused_set)}):")
for key in sorted(unused_set):
print(f" - {key}")
print()
if unused_set and strip_unused:
string_keys = [k for k in string_keys if k not in unused_set]
translations = {
k: v for k, v in translations.items() if k not in unused_set
}
inherited_sets = [s - unused_set for s in inherited_sets]
print(f" Stripping {len(unused_set)} unused string(s) from output.")
out = Path(output_dir)
generate_keys_header(
languages, language_names, string_keys, str(out / "I18nKeys.h")
languages, language_names, string_keys, str(out / "I18nKeys.h"), verbose
)
generate_strings_header(
languages, language_names, str(out / "I18nStrings.h"), verbose
)
generate_strings_header(languages, language_names, str(out / "I18nStrings.h"))
generate_strings_cpp(
languages,
language_names,
string_keys,
translations,
str(out / "I18nStrings.cpp"),
verbose,
)
print()
print("✓ Code generation complete!")
print("Code generation complete!")
print(f" Languages: {len(languages)}")
print(f" String keys: {len(string_keys)}")
if unused_set and not strip_unused:
print(
f" Unused keys: {len(unused_set)} (pass --strip-unused to remove them)"
)
except Exception as e:
print(f"\nError: {e}")
@@ -668,11 +896,52 @@ def main(translations_dir=None, output_dir=None) -> None:
if __name__ == "__main__":
main()
import argparse
parser = argparse.ArgumentParser(
description="Generate I18n C++ files from per-language YAML translations."
)
parser.add_argument(
"translations_dir",
nargs="?",
default=None,
help="Path to the translations directory (default: lib/I18n/translations)",
)
parser.add_argument(
"output_dir",
nargs="?",
default=None,
help="Path to the output directory (default: lib/I18n/)",
)
parser.add_argument(
"--src-dirs",
nargs="+",
metavar="DIR",
default=None,
help="Source directories to scan for STR_* usage (default: src lib)",
)
parser.add_argument(
"--strip-unused",
action="store_true",
help="Remove unused STR_* keys from the generated output",
)
parser.add_argument(
"--verbose",
"-v",
action="store_true",
help="Print per-key INFO/WARNING messages and file generation details",
)
args = parser.parse_args()
main(
args.translations_dir,
args.output_dir,
args.src_dirs,
args.strip_unused,
args.verbose,
)
else:
try:
Import("env")
print("Running i18n generation script from PlatformIO...")
main()
main(strip_unused=True)
except NameError:
pass