Stage scripts

This commit is contained in:
jpirnay
2026-03-30 18:12:52 +02:00
parent 474dfe5490
commit c8b166606e
4 changed files with 9459 additions and 5909 deletions
+143 -47
View File
@@ -3,27 +3,48 @@
// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT. // THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.
// Forward declaration for string arrays // Forward declarations for flat string data blobs and offset tables
namespace i18n_strings { namespace i18n_strings {
extern const char* const STRINGS_EN[]; extern const char STRINGS_EN_DATA[];
extern const char* const STRINGS_ES[]; extern const uint16_t OFFSETS_EN[];
extern const char* const STRINGS_FR[]; extern const char STRINGS_ES_DATA[];
extern const char* const STRINGS_DE[]; extern const uint16_t OFFSETS_ES[];
extern const char* const STRINGS_CS[]; extern const char STRINGS_FR_DATA[];
extern const char* const STRINGS_PO[]; extern const uint16_t OFFSETS_FR[];
extern const char* const STRINGS_RU[]; extern const char STRINGS_DE_DATA[];
extern const char* const STRINGS_SV[]; extern const uint16_t OFFSETS_DE[];
extern const char* const STRINGS_RO[]; extern const char STRINGS_CS_DATA[];
extern const char* const STRINGS_CA[]; extern const uint16_t OFFSETS_CS[];
extern const char* const STRINGS_UK[]; extern const char STRINGS_PO_DATA[];
extern const char* const STRINGS_BE[]; extern const uint16_t OFFSETS_PO[];
extern const char* const STRINGS_IT[]; extern const char STRINGS_RU_DATA[];
extern const char* const STRINGS_PL[]; extern const uint16_t OFFSETS_RU[];
extern const char* const STRINGS_FI[]; extern const char STRINGS_SV_DATA[];
extern const char* const STRINGS_DA[]; extern const uint16_t OFFSETS_SV[];
extern const char* const STRINGS_NL[]; extern const char STRINGS_RO_DATA[];
extern const char* const STRINGS_TR[]; extern const uint16_t OFFSETS_RO[];
extern const char* const STRINGS_KK[]; extern const char STRINGS_CA_DATA[];
extern const uint16_t OFFSETS_CA[];
extern const char STRINGS_UK_DATA[];
extern const uint16_t OFFSETS_UK[];
extern const char STRINGS_BE_DATA[];
extern const uint16_t OFFSETS_BE[];
extern const char STRINGS_IT_DATA[];
extern const uint16_t OFFSETS_IT[];
extern const char STRINGS_PL_DATA[];
extern const uint16_t OFFSETS_PL[];
extern const char STRINGS_FI_DATA[];
extern const uint16_t OFFSETS_FI[];
extern const char STRINGS_DA_DATA[];
extern const uint16_t OFFSETS_DA[];
extern const char STRINGS_NL_DATA[];
extern const uint16_t OFFSETS_NL[];
extern const char STRINGS_TR_DATA[];
extern const uint16_t OFFSETS_TR[];
extern const char STRINGS_KK_DATA[];
extern const uint16_t OFFSETS_KK[];
extern const char STRINGS_PT_DATA[];
extern const uint16_t OFFSETS_PT[];
} // namespace i18n_strings } // namespace i18n_strings
// Language enum // Language enum
@@ -33,7 +54,7 @@ enum class Language : uint8_t {
FR = 2, FR = 2,
DE = 3, DE = 3,
CS = 4, CS = 4,
PT = 5, PO = 5,
RU = 6, RU = 6,
SV = 7, SV = 7,
RO = 8, RO = 8,
@@ -47,6 +68,7 @@ enum class Language : uint8_t {
NL = 16, NL = 16,
TR = 17, TR = 17,
KK = 18, KK = 18,
PT = 19,
_COUNT _COUNT
}; };
@@ -126,6 +148,7 @@ enum class StrId : uint16_t {
STR_IMAGES_DISPLAY, STR_IMAGES_DISPLAY,
STR_IMAGES_PLACEHOLDER, STR_IMAGES_PLACEHOLDER,
STR_IMAGES_SUPPRESS, STR_IMAGES_SUPPRESS,
STR_CREATE_FALLBACK_FOR_INVALID_TOC,
STR_SHORT_PWR_BTN, STR_SHORT_PWR_BTN,
STR_ORIENTATION, STR_ORIENTATION,
STR_SIDE_BTN_LAYOUT, STR_SIDE_BTN_LAYOUT,
@@ -138,6 +161,43 @@ enum class StrId : uint16_t {
STR_HYPHENATION, STR_HYPHENATION,
STR_TIME_TO_SLEEP, STR_TIME_TO_SLEEP,
STR_SHOW_HIDDEN_FILES, STR_SHOW_HIDDEN_FILES,
STR_USE_CLOCK,
STR_CLOCK_SETTINGS,
STR_CLOCK_SETTINGS_WARNING,
STR_CLOCK,
STR_CLOCK_FORMAT,
STR_TIMEZONE,
STR_24H,
STR_12H,
STR_TZ_UTC,
STR_TZ_CET,
STR_TZ_EET,
STR_TZ_EST,
STR_TZ_CST,
STR_TZ_MST,
STR_TZ_PST,
STR_TZ_AEST,
STR_TZ_NZST,
STR_TZ_MSK,
STR_TZ_UTC_MINUS3,
STR_TZ_UTC_PLUS4,
STR_TZ_IST,
STR_TZ_UTC_PLUS7,
STR_TZ_UTC_PLUS8,
STR_TZ_UTC_PLUS9,
STR_SYNC_TIME,
STR_DETECT_TIMEZONE,
STR_SYNCING_CLOCK,
STR_DETECTING_TIMEZONE,
STR_TIME_SYNCED,
STR_TIME_SYNC_FAILED,
STR_CLOCK_DRIFT,
STR_LAST_NTP_SYNC,
STR_TIMEZONE_DETECTED,
STR_TIMEZONE_DETECT_FAILED,
STR_DST_ACTIVE,
STR_DST_INACTIVE,
STR_DST_UNKNOWN,
STR_REFRESH_FREQ, STR_REFRESH_FREQ,
STR_KOREADER_SYNC, STR_KOREADER_SYNC,
STR_CHECK_UPDATES, STR_CHECK_UPDATES,
@@ -159,6 +219,11 @@ enum class StrId : uint16_t {
STR_KOREADER_AUTH, STR_KOREADER_AUTH,
STR_SYNC_READY, STR_SYNC_READY,
STR_AUTH_FAILED, STR_AUTH_FAILED,
STR_REGISTER,
STR_REGISTERING,
STR_REGISTER_SUCCESS,
STR_REGISTER_FAILED,
STR_USERNAME_TAKEN,
STR_DONE, STR_DONE,
STR_CLEAR_CACHE_WARNING_1, STR_CLEAR_CACHE_WARNING_1,
STR_CLEAR_CACHE_WARNING_2, STR_CLEAR_CACHE_WARNING_2,
@@ -286,6 +351,8 @@ enum class StrId : uint16_t {
STR_REMAP_FRONT_BUTTONS, STR_REMAP_FRONT_BUTTONS,
STR_OPDS_BROWSER, STR_OPDS_BROWSER,
STR_COVER_CUSTOM, STR_COVER_CUSTOM,
STR_PAGE_OVERLAY,
STR_RECENTS,
STR_MENU_RECENT_BOOKS, STR_MENU_RECENT_BOOKS,
STR_NO_RECENT_BOOKS, STR_NO_RECENT_BOOKS,
STR_CALIBRE_DESC, STR_CALIBRE_DESC,
@@ -310,6 +377,7 @@ enum class StrId : uint16_t {
STR_SYNC_PROGRESS, STR_SYNC_PROGRESS,
STR_DELETE_CACHE, STR_DELETE_CACHE,
STR_DELETE, STR_DELETE,
STR_REMOVE,
STR_DISPLAY_QR, STR_DISPLAY_QR,
STR_CHAPTER_PREFIX, STR_CHAPTER_PREFIX,
STR_PAGES_SEPARATOR, STR_PAGES_SEPARATOR,
@@ -320,6 +388,8 @@ enum class StrId : uint16_t {
STR_CALC_HASH, STR_CALC_HASH,
STR_HASH_FAILED, STR_HASH_FAILED,
STR_FETCH_PROGRESS, STR_FETCH_PROGRESS,
STR_MAPPING_REMOTE,
STR_MAPPING_LOCAL,
STR_UPLOAD_PROGRESS, STR_UPLOAD_PROGRESS,
STR_NO_CREDENTIALS_MSG, STR_NO_CREDENTIALS_MSG,
STR_KOREADER_SETUP_HINT, STR_KOREADER_SETUP_HINT,
@@ -346,6 +416,10 @@ enum class StrId : uint16_t {
STR_SCREENSHOT_BUTTON, STR_SCREENSHOT_BUTTON,
STR_AUTO_TURN_ENABLED, STR_AUTO_TURN_ENABLED,
STR_AUTO_TURN_PAGES_PER_MIN, STR_AUTO_TURN_PAGES_PER_MIN,
STR_INFO,
STR_AUTHOR,
STR_SERIES,
STR_FILE_SIZE,
STR_SLEEP_COVER_OVERLAY, STR_SLEEP_COVER_OVERLAY,
STR_OVERLAY_WHITE, STR_OVERLAY_WHITE,
STR_OVERLAY_GRAY, STR_OVERLAY_GRAY,
@@ -354,53 +428,76 @@ enum class StrId : uint16_t {
STR_OVERLAY_READING_PROGRESS, STR_OVERLAY_READING_PROGRESS,
STR_OVERLAY_READING_PROGRESS_NO_TOTAL, STR_OVERLAY_READING_PROGRESS_NO_TOTAL,
STR_OVERLAY_CHAPTER_PAGE_SUFFIX, STR_OVERLAY_CHAPTER_PAGE_SUFFIX,
STR_SYSTEM_INFO,
STR_LOAD_XTC_FAILED,
STR_LOAD_EPUB_FAILED,
STR_FW_VERSION,
STR_CHIP,
STR_CPU,
STR_MHZ,
STR_FREE_RAM,
STR_MIN_FREE,
STR_MAX_BLOCK,
STR_FLASH_USED,
STR_UPTIME,
STR_CHARGING,
STR_GATHERING_DATA,
STR_READING,
// Sentinel - must be last // Sentinel - must be last
_COUNT _COUNT
}; };
// Helper function to get string array for a language // Holds a flat string blob and its offset table for one language
inline const char* const* getStringArray(Language lang) { struct LangStrings {
const char* data;
const uint16_t* offsets;
};
// Helper function to get string data for a language
inline LangStrings getLanguageStrings(Language lang) {
switch (lang) { switch (lang) {
case Language::EN: case Language::EN:
return i18n_strings::STRINGS_EN; return {i18n_strings::STRINGS_EN_DATA, i18n_strings::OFFSETS_EN};
case Language::ES: case Language::ES:
return i18n_strings::STRINGS_ES; return {i18n_strings::STRINGS_ES_DATA, i18n_strings::OFFSETS_ES};
case Language::FR: case Language::FR:
return i18n_strings::STRINGS_FR; return {i18n_strings::STRINGS_FR_DATA, i18n_strings::OFFSETS_FR};
case Language::DE: case Language::DE:
return i18n_strings::STRINGS_DE; return {i18n_strings::STRINGS_DE_DATA, i18n_strings::OFFSETS_DE};
case Language::CS: case Language::CS:
return i18n_strings::STRINGS_CS; return {i18n_strings::STRINGS_CS_DATA, i18n_strings::OFFSETS_CS};
case Language::PT: case Language::PO:
return i18n_strings::STRINGS_PO; return {i18n_strings::STRINGS_PO_DATA, i18n_strings::OFFSETS_PO};
case Language::RU: case Language::RU:
return i18n_strings::STRINGS_RU; return {i18n_strings::STRINGS_RU_DATA, i18n_strings::OFFSETS_RU};
case Language::SV: case Language::SV:
return i18n_strings::STRINGS_SV; return {i18n_strings::STRINGS_SV_DATA, i18n_strings::OFFSETS_SV};
case Language::RO: case Language::RO:
return i18n_strings::STRINGS_RO; return {i18n_strings::STRINGS_RO_DATA, i18n_strings::OFFSETS_RO};
case Language::CA: case Language::CA:
return i18n_strings::STRINGS_CA; return {i18n_strings::STRINGS_CA_DATA, i18n_strings::OFFSETS_CA};
case Language::UK: case Language::UK:
return i18n_strings::STRINGS_UK; return {i18n_strings::STRINGS_UK_DATA, i18n_strings::OFFSETS_UK};
case Language::BE: case Language::BE:
return i18n_strings::STRINGS_BE; return {i18n_strings::STRINGS_BE_DATA, i18n_strings::OFFSETS_BE};
case Language::IT: case Language::IT:
return i18n_strings::STRINGS_IT; return {i18n_strings::STRINGS_IT_DATA, i18n_strings::OFFSETS_IT};
case Language::PL: case Language::PL:
return i18n_strings::STRINGS_PL; return {i18n_strings::STRINGS_PL_DATA, i18n_strings::OFFSETS_PL};
case Language::FI: case Language::FI:
return i18n_strings::STRINGS_FI; return {i18n_strings::STRINGS_FI_DATA, i18n_strings::OFFSETS_FI};
case Language::DA: case Language::DA:
return i18n_strings::STRINGS_DA; return {i18n_strings::STRINGS_DA_DATA, i18n_strings::OFFSETS_DA};
case Language::NL: case Language::NL:
return i18n_strings::STRINGS_NL; return {i18n_strings::STRINGS_NL_DATA, i18n_strings::OFFSETS_NL};
case Language::TR: case Language::TR:
return i18n_strings::STRINGS_TR; return {i18n_strings::STRINGS_TR_DATA, i18n_strings::OFFSETS_TR};
case Language::KK: case Language::KK:
return i18n_strings::STRINGS_KK; return {i18n_strings::STRINGS_KK_DATA, i18n_strings::OFFSETS_KK};
case Language::PT:
return {i18n_strings::STRINGS_PT_DATA, i18n_strings::OFFSETS_PT};
default: default:
return i18n_strings::STRINGS_EN; return {i18n_strings::STRINGS_EN_DATA, i18n_strings::OFFSETS_EN};
} }
} }
@@ -408,9 +505,8 @@ inline const char* const* getStringArray(Language lang) {
constexpr uint8_t getLanguageCount() { return static_cast<uint8_t>(Language::_COUNT); } constexpr uint8_t getLanguageCount() { return static_cast<uint8_t>(Language::_COUNT); }
// Sorted language indices by code (auto-generated by gen_i18n.py) // Sorted language indices by code (auto-generated by gen_i18n.py)
// Order: English, Беларуская, Català, Čeština, Dansk, Deutsch, Español, Suomi, Français, Italiano // Order: English, Беларуская, Català, Čeština, Dansk, Deutsch, Español, Suomi, Français, Italiano, Қазақша, Nederlands, Polski, Português (Brasil), Português (Portugal), Română, Русский, Svenska, Türkçe, Українська
// Қазақша, Nederlands, Polski, Português (Brasil), Română, Русский, Svenska, Türkçe, Українська constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {0, 11, 9, 4, 15, 3, 1, 14, 2, 12, 18, 16, 13, 5, 19, 8, 6, 7, 17, 10};
constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {0, 11, 9, 4, 15, 3, 1, 14, 2, 12, 18, 16, 13, 5, 8, 6, 7, 17, 10};
static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount(), static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) == getLanguageCount(),
"SORTED_LANGUAGE_INDICES size mismatch"); "SORTED_LANGUAGE_INDICES size mismatch");
+8893 -5727
View File
File diff suppressed because it is too large Load Diff
+40 -21
View File
@@ -1,30 +1,49 @@
#pragma once #pragma once
#include <string>
#include "I18nKeys.h" #include "I18nKeys.h"
// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT. // THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.
namespace i18n_strings { namespace i18n_strings {
extern const char* const STRINGS_EN[]; extern const char STRINGS_EN_DATA[];
extern const char* const STRINGS_ES[]; extern const uint16_t OFFSETS_EN[];
extern const char* const STRINGS_FR[]; extern const char STRINGS_ES_DATA[];
extern const char* const STRINGS_DE[]; extern const uint16_t OFFSETS_ES[];
extern const char* const STRINGS_CS[]; extern const char STRINGS_FR_DATA[];
extern const char* const STRINGS_PO[]; extern const uint16_t OFFSETS_FR[];
extern const char* const STRINGS_RU[]; extern const char STRINGS_DE_DATA[];
extern const char* const STRINGS_SV[]; extern const uint16_t OFFSETS_DE[];
extern const char* const STRINGS_RO[]; extern const char STRINGS_CS_DATA[];
extern const char* const STRINGS_CA[]; extern const uint16_t OFFSETS_CS[];
extern const char* const STRINGS_UK[]; extern const char STRINGS_PO_DATA[];
extern const char* const STRINGS_BE[]; extern const uint16_t OFFSETS_PO[];
extern const char* const STRINGS_IT[]; extern const char STRINGS_RU_DATA[];
extern const char* const STRINGS_PL[]; extern const uint16_t OFFSETS_RU[];
extern const char* const STRINGS_FI[]; extern const char STRINGS_SV_DATA[];
extern const char* const STRINGS_DA[]; extern const uint16_t OFFSETS_SV[];
extern const char* const STRINGS_NL[]; extern const char STRINGS_RO_DATA[];
extern const char* const STRINGS_TR[]; extern const uint16_t OFFSETS_RO[];
extern const char* const STRINGS_KK[]; extern const char STRINGS_CA_DATA[];
extern const uint16_t OFFSETS_CA[];
extern const char STRINGS_UK_DATA[];
extern const uint16_t OFFSETS_UK[];
extern const char STRINGS_BE_DATA[];
extern const uint16_t OFFSETS_BE[];
extern const char STRINGS_IT_DATA[];
extern const uint16_t OFFSETS_IT[];
extern const char STRINGS_PL_DATA[];
extern const uint16_t OFFSETS_PL[];
extern const char STRINGS_FI_DATA[];
extern const uint16_t OFFSETS_FI[];
extern const char STRINGS_DA_DATA[];
extern const uint16_t OFFSETS_DA[];
extern const char STRINGS_NL_DATA[];
extern const uint16_t OFFSETS_NL[];
extern const char STRINGS_TR_DATA[];
extern const uint16_t OFFSETS_TR[];
extern const char STRINGS_KK_DATA[];
extern const uint16_t OFFSETS_KK[];
extern const char STRINGS_PT_DATA[];
extern const uint16_t OFFSETS_PT[];
} // namespace i18n_strings } // namespace i18n_strings
+383 -114
View File
@@ -15,18 +15,25 @@ Each YAML file must contain:
The English file is the reference. Missing keys in other languages are The English file is the reference. Missing keys in other languages are
automatically filled from English, with a warning. automatically filled from English, with a warning.
Usage: By default the script scans the src/ and lib/ trees for STR_* references and
python gen_i18n.py <translations_dir> <output_dir> reports any translation keys that are never used. Pass --strip-unused to
omit those keys from the generated output entirely.
Example: Usage:
python gen_i18n.py [translations_dir [output_dir]] [options]
Examples:
python gen_i18n.py
python gen_i18n.py lib/I18n/translations lib/I18n/ python gen_i18n.py lib/I18n/translations lib/I18n/
python gen_i18n.py --strip-unused
python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont
""" """
import sys import sys
import os import os
import re import re
from pathlib import Path from pathlib import Path
from typing import List, Dict, Tuple from typing import Dict, List, Optional, Set, Tuple
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -105,7 +112,8 @@ def parse_yaml_file(filepath: str) -> Dict[str, str]:
def load_translations( def load_translations(
translations_dir: str, translations_dir: str,
) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]]]: verbose: bool = False,
) -> Tuple[List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]:
""" """
Read every YAML file in *translations_dir* and return: Read every YAML file in *translations_dir* and return:
language_codes e.g. ["EN", "ES", ...] language_codes e.g. ["EN", "ES", ...]
@@ -138,6 +146,31 @@ def load_translations(
if english_file is None: if english_file is None:
raise ValueError("No YAML file with _language_code: EN found") raise ValueError("No YAML file with _language_code: EN found")
duplicate_orders: Dict[str, List[str]] = {}
order_to_files: Dict[str, List[str]] = {}
for fname, data in parsed.items():
order = data.get("_order")
if not order:
continue
order_to_files.setdefault(order, []).append(fname)
for order, files in order_to_files.items():
if len(files) > 1:
duplicate_orders[order] = sorted(files)
if duplicate_orders:
duplicate_messages = [
f"_order {order}: {', '.join(files)}"
for order, files in sorted(
duplicate_orders.items(), key=lambda item: int(item[0])
)
]
raise ValueError(
"Duplicate _order values found:\n "
+ "\n ".join(duplicate_messages)
+ "\nEach _order value must be unique to ensure a deterministic language order."
)
# Order: English first, then by _order metadata (falls back to filename) # Order: English first, then by _order metadata (falls back to filename)
def sort_key(fname: str) -> Tuple[int, int, str]: def sort_key(fname: str) -> Tuple[int, int, str]:
"""English always first (0), then by _order, then by filename.""" """English always first (0), then by _order, then by filename."""
@@ -174,16 +207,20 @@ def load_translations(
raise ValueError(f"Invalid C++ identifier in English file: '{key}'") raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
# Build translations dict, filling missing keys from English # Build translations dict, filling missing keys from English
inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
translations: Dict[str, List[str]] = {} translations: Dict[str, List[str]] = {}
for key in string_keys: for key in string_keys:
row: List[str] = [] row: List[str] = []
for fname in ordered_files: for lang_idx, fname in enumerate(ordered_files):
data = parsed[fname] data = parsed[fname]
value = data.get(key, "") value = data.get(key, "")
if not value.strip() and fname != english_file: if not value.strip() and fname != english_file:
value = english_data[key] value = english_data[key]
lang_code = parsed[fname].get("_language_code", fname) inherited_sets[lang_idx].add(key)
print(f" INFO: '{key}' missing in {lang_code}, using English fallback") if verbose:
print(
f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
)
row.append(value) row.append(value)
translations[key] = row translations[key] = row
@@ -195,59 +232,72 @@ def load_translations(
extra = [k for k in data if not k.startswith("_") and k not in english_data] extra = [k for k in data if not k.startswith("_") and k not in english_data]
if extra: if extra:
lang_code = data.get("_language_code", fname) lang_code = data.get("_language_code", fname)
print(f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}") if verbose:
print(
f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}"
)
print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") if verbose:
return language_codes, language_names, string_keys, translations print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys")
return language_codes, language_names, string_keys, translations, inherited_sets
# ---------------------------------------------------------------------------
# Unused-string detection
# ---------------------------------------------------------------------------
_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"}
def find_used_string_keys(
src_dirs: List[str],
skip_filenames: Optional[Set[str]] = None,
) -> Set[str]:
"""
Scan C/C++ source files under *src_dirs* for STR_* identifiers.
Files whose basename appears in *skip_filenames* are skipped so that
the generated I18n files don't count as "usage" of themselves.
Returns the set of all STR_KEY names that appear at least once.
"""
if skip_filenames is None:
skip_filenames = _GENERATED_FILENAMES
pattern = re.compile(r"\bSTR_[A-Za-z0-9_]+\b")
used: Set[str] = set()
for src_dir in src_dirs:
p = Path(src_dir)
if not p.is_dir():
continue
for f in p.rglob("*"):
if f.suffix not in {".cpp", ".h", ".c"}:
continue
if f.name in skip_filenames:
continue
try:
text = f.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
for m in pattern.finditer(text):
used.add(m.group(0))
return used
def report_unused_keys(
string_keys: List[str],
used_keys: Set[str],
) -> List[str]:
"""Return a sorted list of keys from *string_keys* absent in *used_keys*."""
return [k for k in sorted(string_keys) if k not in used_keys]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# C++ string escaping # C++ string escaping
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
LANG_ABBREVIATIONS = {
"english": "EN",
"español": "ES",
"espanol": "ES",
"italiano": "IT",
"svenska": "SV",
"français": "FR",
"francais": "FR",
"deutsch": "DE",
"german": "DE",
"polski": "PL",
"português": "PT",
"portugues": "PT",
"português (brasil)": "PO",
"中文": "ZH",
"chinese": "ZH",
"日本語": "JA",
"japanese": "JA",
"한국어": "KO",
"korean": "KO",
"русский": "RU",
"russian": "RU",
"العربية": "AR",
"arabic": "AR",
"עברית": "HE",
"hebrew": "HE",
"فارسی": "FA",
"persian": "FA",
"čeština": "CS",
"türkçe": "TR",
"turkish": "TR",
"Қазақша": "KK",
"kazakh": "KK",
}
def get_lang_abbreviation(lang_code: str, lang_name: str) -> str:
"""Return a 2-letter abbreviation for a language."""
lower = lang_name.lower()
if lower in LANG_ABBREVIATIONS:
return LANG_ABBREVIATIONS[lower]
return lang_code[:2].upper()
def escape_cpp_string(s: str) -> List[str]: def escape_cpp_string(s: str) -> List[str]:
r""" r"""
@@ -390,6 +440,7 @@ def generate_keys_header(
language_names: List[str], language_names: List[str],
string_keys: List[str], string_keys: List[str],
output_path: str, output_path: str,
verbose: bool = False,
) -> None: ) -> None:
"""Generate I18nKeys.h.""" """Generate I18nKeys.h."""
lines: List[str] = [ lines: List[str] = [
@@ -398,13 +449,13 @@ def generate_keys_header(
"", "",
"// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
"", "",
"// Forward declaration for string arrays", "// Forward declarations for flat string data blobs and offset tables",
"namespace i18n_strings {", "namespace i18n_strings {",
] ]
for code, name in zip(languages, language_names): for code in languages:
abbrev = get_lang_abbreviation(code, name) lines.append(f"extern const char STRINGS_{code}_DATA[];")
lines.append(f"extern const char* const STRINGS_{abbrev}[];") lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("} // namespace i18n_strings") lines.append("} // namespace i18n_strings")
lines.append("") lines.append("")
@@ -436,17 +487,28 @@ def generate_keys_header(
lines.append("};") lines.append("};")
lines.append("") lines.append("")
# getStringArray helper # LangStrings struct
lines.append("// Helper function to get string array for a language") lines.append("// Holds a flat string blob and its offset table for one language")
lines.append("inline const char* const* getStringArray(Language lang) {") lines.append("struct LangStrings {")
lines.append(" const char* data;")
lines.append(" const uint16_t* offsets;")
lines.append("};")
lines.append("")
# getLanguageStrings helper
lines.append("// Helper function to get string data for a language")
lines.append("inline LangStrings getLanguageStrings(Language lang) {")
lines.append(" switch (lang) {") lines.append(" switch (lang) {")
for code, name in zip(languages, language_names): for code in languages:
abbrev = get_lang_abbreviation(code, name)
lines.append(f" case Language::{code}:") lines.append(f" case Language::{code}:")
lines.append(f" return i18n_strings::STRINGS_{abbrev};") lines.append(
first_abbrev = get_lang_abbreviation(languages[0], language_names[0]) f" return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};"
)
first_code = languages[0]
lines.append(" default:") lines.append(" default:")
lines.append(f" return i18n_strings::STRINGS_{first_abbrev};") lines.append(
f" return {{i18n_strings::STRINGS_{first_code}_DATA, i18n_strings::OFFSETS_{first_code}}};"
)
lines.append(" }") lines.append(" }")
lines.append("}") lines.append("}")
lines.append("") lines.append("")
@@ -467,21 +529,9 @@ def generate_keys_header(
key=lambda i: languages[i], key=lambda i: languages[i],
) )
sorted_indices = [english_idx] + rest sorted_indices = [english_idx] + rest
names = [language_names[i] for i in sorted_indices] comment_names = ", ".join(language_names[i] for i in sorted_indices)
comment_names = ", ".join(names)
lines.append("// Sorted language indices by code (auto-generated by gen_i18n.py)") lines.append("// Sorted language indices by code (auto-generated by gen_i18n.py)")
if len(f"// Order: {comment_names}") <= 120: lines.append(f"// Order: {comment_names}")
lines.append(f"// Order: {comment_names}")
else:
current = "// Order: "
for name in names:
candidate = current + name + ", "
if len(candidate) > 100:
lines.append(current.rstrip(", "))
current = "// " + name + ", "
else:
current = candidate
lines.append(current.rstrip(", "))
lines.append( lines.append(
"constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {" "constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {"
f"{', '.join(str(i) for i in sorted_indices)}" f"{', '.join(str(i) for i in sorted_indices)}"
@@ -493,19 +543,18 @@ def generate_keys_header(
) )
lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");') lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");')
_write_file(output_path, lines) _write_file(output_path, lines, verbose)
def generate_strings_header( def generate_strings_header(
languages: List[str], languages: List[str],
language_names: List[str], language_names: List[str],
output_path: str, output_path: str,
verbose: bool = False,
) -> None: ) -> None:
"""Generate I18nStrings.h.""" """Generate I18nStrings.h."""
lines: List[str] = [ lines: List[str] = [
"#pragma once", "#pragma once",
"#include <string>",
"",
'#include "I18nKeys.h"', '#include "I18nKeys.h"',
"", "",
"// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
@@ -514,13 +563,13 @@ def generate_strings_header(
"", "",
] ]
for code, name in zip(languages, language_names): for code in languages:
abbrev = get_lang_abbreviation(code, name) lines.append(f"extern const char STRINGS_{code}_DATA[];")
lines.append(f"extern const char* const STRINGS_{abbrev}[];") lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("") lines.append("")
lines.append("} // namespace i18n_strings") lines.append("} // namespace i18n_strings")
_write_file(output_path, lines) _write_file(output_path, lines, verbose)
def generate_strings_cpp( def generate_strings_cpp(
@@ -529,10 +578,12 @@ def generate_strings_cpp(
string_keys: List[str], string_keys: List[str],
translations: Dict[str, List[str]], translations: Dict[str, List[str]],
output_path: str, output_path: str,
verbose: bool = False,
) -> None: ) -> None:
"""Generate I18nStrings.cpp.""" """Generate I18nStrings.cpp."""
lines: List[str] = [ lines: List[str] = [
'#include "I18nStrings.h"', '#include "I18nStrings.h"',
"#include <cstddef>",
"", "",
"// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.", "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT.",
"", "",
@@ -555,18 +606,41 @@ def generate_strings_cpp(
lines.append("};") lines.append("};")
lines.append("") lines.append("")
# Per-language string arrays # Per-language flat string blobs and offset tables
lines.append("namespace i18n_strings {") lines.append("namespace i18n_strings {")
lines.append("") lines.append("")
for lang_idx, (code, name) in enumerate(zip(languages, language_names)): for lang_idx, code in enumerate(languages):
abbrev = get_lang_abbreviation(code, name) lang_strings = [translations[key][lang_idx] for key in string_keys]
lines.append(f"const char* const STRINGS_{abbrev}[] = {{")
for key in string_keys: # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator)
text = translations[key][lang_idx] offsets: List[int] = []
_append_string_entry(lines, text) current_offset = 0
for s in lang_strings:
offsets.append(current_offset)
current_offset += len(s.encode("utf-8")) + 1
if current_offset > 65535:
raise ValueError(
f"Language {code}: total string data ({current_offset} bytes) "
"exceeds uint16_t offset range (65535)"
)
# Flat string data blob — all strings concatenated with \0 separators.
# clang-format off/on avoids reformatting of the adjacent string literals.
lines.append("// clang-format off")
lines.append(f"const char STRINGS_{code}_DATA[] =")
for text in lang_strings:
_append_string_data_entry(lines, text)
lines.append(";")
lines.append("// clang-format on")
lines.append("")
# Offset table — one uint16_t per StrId
lines.append(f"const uint16_t OFFSETS_{code}[] = {{")
chunk_size = 12
for i in range(0, len(offsets), chunk_size):
chunk = offsets[i : i + chunk_size]
lines.append(" " + ", ".join(str(o) for o in chunk) + ",")
lines.append("};") lines.append("};")
lines.append("") lines.append("")
@@ -575,16 +649,15 @@ def generate_strings_cpp(
# Compile-time size checks # Compile-time size checks
lines.append("// Compile-time validation of array sizes") lines.append("// Compile-time validation of array sizes")
for code, name in zip(languages, language_names): for code in languages:
abbrev = get_lang_abbreviation(code, name)
lines.append( lines.append(
f"static_assert(sizeof(i18n_strings::STRINGS_{abbrev}) " f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) "
f"/ sizeof(i18n_strings::STRINGS_{abbrev}[0]) ==" f"/ sizeof(i18n_strings::OFFSETS_{code}[0]) =="
) )
lines.append(" static_cast<size_t>(StrId::_COUNT),") lines.append(" static_cast<size_t>(StrId::_COUNT),")
lines.append(f' "STRINGS_{abbrev} size mismatch");') lines.append(f' "OFFSETS_{code} size mismatch");')
_write_file(output_path, lines) _write_file(output_path, lines, verbose)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -592,6 +665,91 @@ def generate_strings_cpp(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _print_language_table(
language_codes: List[str],
language_names: List[str],
inherited_sets: List[Set[str]],
string_keys: List[str],
unused_keys: Set[str],
data_sizes: List[int],
) -> None:
"""Print a per-language summary table."""
total = len(string_keys)
headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)")
rows = []
for code, name, inherited, size in zip(
language_codes, language_names, inherited_sets, data_sizes
):
own = total - len(inherited)
fallback = len(inherited)
# strings this language translated but the code never calls
unused = len(unused_keys - inherited)
rows.append((name, code, str(own), str(fallback), str(unused), str(size)))
# EN first, then alphabetically by ISO code
rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
col_widths = [len(h) for h in headers]
for row in rows:
for i, cell in enumerate(row):
col_widths[i] = max(col_widths[i], len(cell))
fmt = " ".join(f"{{:<{w}}}" for w in col_widths)
sep = " ".join("-" * w for w in col_widths)
def _safe_print(line: str) -> None:
print(
line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode(
sys.stdout.encoding or "utf-8", errors="replace"
)
)
_safe_print(fmt.format(*headers))
_safe_print(sep)
for row in rows:
_safe_print(fmt.format(*row))
used = total - len(unused_keys)
total_size = sum(data_sizes)
n_lang = len(rows)
n_keys = len(string_keys)
# Current layout: uint16_t offset table (2 B per string per language)
offset_table_size = n_lang * n_keys * 2
current_total = total_size + offset_table_size
# Previous layout: const char* pointer array (4 B per string per language)
old_pointer_table_size = n_lang * n_keys * 4
old_total = total_size + old_pointer_table_size
saved = old_total - current_total
print(
f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}"
)
print(
f" Flash (now): {total_size:>7,} B strings + {offset_table_size:>6,} B offset tables (uint16_t)"
f" = {current_total:>7,} B"
)
print(
f" Flash (before): {total_size:>7,} B strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)"
f" = {old_total:>7,} B"
)
print(f" Saved by offset tables: {saved:,} B")
def _append_string_data_entry(lines: List[str], text: str) -> None:
"""
Escape *text*, append a \\0 null separator, and format as indented C++
string literal lines for inclusion in a flat char data array blob.
"""
segments = escape_cpp_string(text)
# Append the null entry separator to the last segment
if segments and segments[-1] != "":
segments[-1] += "\\0"
elif segments:
segments[-1] = "\\0"
else:
segments = ["\\0"]
lines.extend(format_cpp_string_literal(segments))
def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None: def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None:
"""Escape *text*, format as indented C++ lines, append comma (and optional comment).""" """Escape *text*, format as indented C++ lines, append comma (and optional comment)."""
segments = escape_cpp_string(text) segments = escape_cpp_string(text)
@@ -601,11 +759,12 @@ def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None
lines.extend(formatted) lines.extend(formatted)
def _write_file(path: str, lines: List[str]) -> None: def _write_file(path: str, lines: List[str], verbose: bool = False) -> None:
with open(path, "w", encoding="utf-8", newline="\n") as f: with open(path, "w", encoding="utf-8", newline="\n") as f:
f.write("\n".join(lines)) f.write("\n".join(lines))
f.write("\n") f.write("\n")
print(f"Generated: {path}") if verbose:
print(f"Generated: {path}")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -613,10 +772,17 @@ def _write_file(path: str, lines: List[str]) -> None:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def main(translations_dir=None, output_dir=None) -> None: def main(
translations_dir: Optional[str] = None,
output_dir: Optional[str] = None,
src_dirs: Optional[List[str]] = None,
strip_unused: bool = False,
verbose: bool = False,
) -> None:
# Default paths (relative to project root) # Default paths (relative to project root)
default_translations_dir = "lib/I18n/translations" default_translations_dir = "lib/I18n/translations"
default_output_dir = "lib/I18n/" default_output_dir = "lib/I18n/"
default_src_dirs = ["src", "lib"]
if translations_dir is None or output_dir is None: if translations_dir is None or output_dir is None:
if len(sys.argv) == 3: if len(sys.argv) == 3:
@@ -627,6 +793,9 @@ def main(translations_dir=None, output_dir=None) -> None:
translations_dir = default_translations_dir translations_dir = default_translations_dir
output_dir = default_output_dir output_dir = default_output_dir
if src_dirs is None:
src_dirs = default_src_dirs
if not os.path.isdir(translations_dir): if not os.path.isdir(translations_dir):
print(f"Error: Translations directory not found: {translations_dir}") print(f"Error: Translations directory not found: {translations_dir}")
sys.exit(1) sys.exit(1)
@@ -635,32 +804,91 @@ def main(translations_dir=None, output_dir=None) -> None:
print(f"Error: Output directory not found: {output_dir}") print(f"Error: Output directory not found: {output_dir}")
sys.exit(1) sys.exit(1)
print(f"Reading translations from: {translations_dir}") if verbose:
print(f"Output directory: {output_dir}") print(f"Reading translations from: {translations_dir}")
print() print(f"Output directory: {output_dir}")
print()
try: try:
languages, language_names, string_keys, translations = load_translations( languages, language_names, string_keys, translations, inherited_sets = (
translations_dir load_translations(translations_dir, verbose)
) )
# --- Unused-string detection ---
scan_dirs = [d for d in src_dirs if os.path.isdir(d)]
if scan_dirs:
used_keys = find_used_string_keys(scan_dirs)
unused_set = set(report_unused_keys(string_keys, used_keys))
else:
used_keys = set(string_keys)
unused_set = set()
# --- Missing-string detection (used in code but absent from English) ---
missing_keys = sorted(used_keys - set(string_keys))
if missing_keys:
print(
f"\n CRITICAL: {len(missing_keys)} string(s) used in source but missing from english.yaml:"
)
for key in missing_keys:
print(f" - {key}")
print()
sys.exit(1)
# Compute per-language data blob sizes:
# sum of UTF-8 byte length + 1 (null terminator) per string
data_sizes = [
sum(len(translations[k][i].encode("utf-8")) + 1 for k in string_keys)
for i in range(len(languages))
]
_print_language_table(
languages,
language_names,
inherited_sets,
string_keys,
unused_set,
data_sizes,
)
print()
if verbose and unused_set:
print(f" Unused keys ({len(unused_set)}):")
for key in sorted(unused_set):
print(f" - {key}")
print()
if unused_set and strip_unused:
string_keys = [k for k in string_keys if k not in unused_set]
translations = {
k: v for k, v in translations.items() if k not in unused_set
}
inherited_sets = [s - unused_set for s in inherited_sets]
print(f" Stripping {len(unused_set)} unused string(s) from output.")
out = Path(output_dir) out = Path(output_dir)
generate_keys_header( generate_keys_header(
languages, language_names, string_keys, str(out / "I18nKeys.h") languages, language_names, string_keys, str(out / "I18nKeys.h"), verbose
)
generate_strings_header(
languages, language_names, str(out / "I18nStrings.h"), verbose
) )
generate_strings_header(languages, language_names, str(out / "I18nStrings.h"))
generate_strings_cpp( generate_strings_cpp(
languages, languages,
language_names, language_names,
string_keys, string_keys,
translations, translations,
str(out / "I18nStrings.cpp"), str(out / "I18nStrings.cpp"),
verbose,
) )
print() print()
print("Code generation complete!") print("Code generation complete!")
print(f" Languages: {len(languages)}") print(f" Languages: {len(languages)}")
print(f" String keys: {len(string_keys)}") print(f" String keys: {len(string_keys)}")
if unused_set and not strip_unused:
print(
f" Unused keys: {len(unused_set)} (pass --strip-unused to remove them)"
)
except Exception as e: except Exception as e:
print(f"\nError: {e}") print(f"\nError: {e}")
@@ -668,11 +896,52 @@ def main(translations_dir=None, output_dir=None) -> None:
if __name__ == "__main__": if __name__ == "__main__":
main() import argparse
parser = argparse.ArgumentParser(
description="Generate I18n C++ files from per-language YAML translations."
)
parser.add_argument(
"translations_dir",
nargs="?",
default=None,
help="Path to the translations directory (default: lib/I18n/translations)",
)
parser.add_argument(
"output_dir",
nargs="?",
default=None,
help="Path to the output directory (default: lib/I18n/)",
)
parser.add_argument(
"--src-dirs",
nargs="+",
metavar="DIR",
default=None,
help="Source directories to scan for STR_* usage (default: src lib)",
)
parser.add_argument(
"--strip-unused",
action="store_true",
help="Remove unused STR_* keys from the generated output",
)
parser.add_argument(
"--verbose",
"-v",
action="store_true",
help="Print per-key INFO/WARNING messages and file generation details",
)
args = parser.parse_args()
main(
args.translations_dir,
args.output_dir,
args.src_dirs,
args.strip_unused,
args.verbose,
)
else: else:
try: try:
Import("env") Import("env")
print("Running i18n generation script from PlatformIO...") main(strip_unused=True)
main()
except NameError: except NameError:
pass pass