eric/Crosspoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Account for deduplication

Browse Source
This commit is contained in:
jpirnay
2026-05-01 09:25:15 +02:00
parent 65d36d4f24
commit 999cf74c58
2 changed files with 82 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/I18n/I18n.cpp
+15 -12
View File
@@ -1,7 +1,7 @@
#include "I18n.h"
#include <HalStorage.h>
#include <HardwareSerial.h>
#include <Logging.h>
#include <Serialization.h>
#include <string>
@@ -27,7 +27,12 @@ const char* I18n::get(StrId id) const {
// Use generated helper function - no hardcoded switch needed!
const LangStrings lang = getLanguageStrings(_language);
return lang.data + lang.offsets[index];
const char* result = lang.data + lang.offsets[index];
if (_language != Language::EN && result[0] == '\0') {
const LangStrings english = getLanguageStrings(Language::EN);
return english.data + english.offsets[index];
}
return result;
}
void I18n::setLanguage(Language lang) {
@@ -59,7 +64,7 @@ void I18n::saveSettings() {
FsFile file;
if (!Storage.openFileForWrite("I18N", SETTINGS_FILE, file)) {
Serial.printf("[I18N] Failed to save settings\n");
LOG_ERR("I18N", "Failed to save settings");
return;
}
@@ -67,14 +72,13 @@ void I18n::saveSettings() {
serialization::writeString(file, getLanguageCode(_language));
file.close();
Serial.printf("[I18N] Settings saved: language=%d code=%s\n", static_cast<int>(_language),
getLanguageCode(_language));
LOG_INF("I18N", "Settings saved: language=%d code=%s", static_cast<int>(_language), getLanguageCode(_language));
}
void I18n::loadSettings() {
FsFile file;
if (!Storage.openFileForRead("I18N", SETTINGS_FILE, file)) {
Serial.printf("[I18N] No settings file, using default (English)\n");
LOG_INF("I18N", "No settings file, using default (English)");
return;
}
@@ -95,9 +99,9 @@ void I18n::loadSettings() {
}
if (found) {
Serial.printf("[I18N] Loaded language code: %s (%d)\n", code.c_str(), static_cast<int>(_language));
LOG_INF("I18N", "Loaded language code: %s (%d)", code.c_str(), static_cast<int>(_language));
} else {
Serial.printf("[I18N] Unknown language code in settings: %s\n", code.c_str());
LOG_ERR("I18N", "Unknown language code in settings: %s", code.c_str());
}
file.close();
return;
@@ -109,18 +113,17 @@ void I18n::loadSettings() {
serialization::readPod(file, lang);
if (lang < static_cast<size_t>(Language::_COUNT)) {
_language = static_cast<Language>(lang);
Serial.printf("[I18N] Migrating v1 language index: %d -> %s\n", static_cast<int>(_language),
getLanguageCode(_language));
LOG_INF("I18N", "Migrating v1 language index: %d -> %s", static_cast<int>(_language), getLanguageCode(_language));
file.close();
saveSettings();
return;
}
file.close();
Serial.printf("[I18N] Invalid v1 language index: %d\n", static_cast<int>(lang));
LOG_ERR("I18N", "Invalid v1 language index: %d", static_cast<int>(lang));
return;
}
Serial.printf("[I18N] Settings version mismatch: %d\n", static_cast<int>(version));
LOG_ERR("I18N", "Settings version mismatch: %d\n", static_cast<int>(version));
file.close();
}
scripts/gen_i18n.py
+62 -13
View File
@@ -206,16 +206,31 @@ def load_translations(
if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", key):
raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
# Build translations dict, filling missing keys from English
# Build translations dict, filling missing keys from English.
# Non-English values that are absent or blank fall back to English at runtime.
# A literal empty string in a non-English YAML file is treated as an intentional
# empty translation and stored as a single space to distinguish it from fallback.
inherited_sets: List[Set[str]] = [set() for _ in ordered_files]
translations: Dict[str, List[str]] = {}
for key in string_keys:
row: List[str] = []
for lang_idx, fname in enumerate(ordered_files):
data = parsed[fname]
value = data.get(key, "")
if not value.strip() and fname != english_file:
value = english_data[key]
if key not in data:
value = ""
if fname != english_file:
inherited_sets[lang_idx].add(key)
if verbose:
print(
f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback"
)
else:
value = data[key]
if fname != english_file:
if value == "":
value = " "
elif not value.strip():
value = ""
inherited_sets[lang_idx].add(key)
if verbose:
print(
@@ -474,8 +489,12 @@ def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List
def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str:
"""Return a sorted string of every unique character used in a language."""
chars = set()
english_index = 0
for values in translations.values():
for ch in values[lang_index]:
text = values[lang_index]
if lang_index != english_index and text == "":
text = values[english_index]
for ch in text:
chars.add(ord(ch))
return "".join(chr(cp) for cp in sorted(chars))
@@ -741,20 +760,43 @@ def _print_language_table(
string_keys: List[str],
unused_keys: Set[str],
data_sizes: List[int],
translations: Dict[str, List[str]],
) -> None:
"""Print a per-language summary table."""
total = len(string_keys)
headers = ("Language", "Code", "Own", "Fallback", "Unused", "Data (B)")
headers = (
"Language",
"Code",
"Own",
"Fallback",
"Unused",
"Data (B)",
"Unique (B)",
)
rows = []
for code, name, inherited, size in zip(
language_codes, language_names, inherited_sets, data_sizes
for lang_idx, (code, name, inherited, size) in enumerate(
zip(language_codes, language_names, inherited_sets, data_sizes)
):
own = total - len(inherited)
fallback = len(inherited)
# strings this language translated but the code never calls
unused = len(unused_keys - inherited)
rows.append((name, code, str(own), str(fallback), str(unused), str(size)))
unique_size = sum(
len(s.encode("utf-8")) + 1
for s in {translations[k][lang_idx] for k in string_keys}
)
rows.append(
(
name,
code,
str(own),
str(fallback),
str(unused),
str(size),
str(unique_size),
)
)
# EN first, then alphabetically by ISO code
rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[1]))
@@ -785,10 +827,16 @@ def _print_language_table(
# Current layout: uint16_t offset table (2 B per string per language)
offset_table_size = n_lang * n_keys * 2
current_total = total_size + offset_table_size
# Previous layout: const char* pointer array (4 B per string per language)
# Estimate the original pointer-based layout using deduplicated string storage.
unique_strings: Set[str] = set()
for values in translations.values():
unique_strings.update(values)
unique_string_size = sum(len(s.encode("utf-8")) + 1 for s in unique_strings)
old_pointer_table_size = n_lang * n_keys * 4
old_total = total_size + old_pointer_table_size
old_total = unique_string_size + old_pointer_table_size
saved = old_total - current_total
print(
f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}"
)
@@ -797,10 +845,10 @@ def _print_language_table(
f" = {current_total:>7,} B"
)
print(
f" Flash (before): {total_size:>7,} B strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)"
f" Flash (pointer model, deduped): {unique_string_size:>7,} B unique strings + {old_pointer_table_size:>6,} B pointer tables (ptr32)"
f" = {old_total:>7,} B"
)
print(f" Saved by offset tables: {saved:,} B")
print(f" Estimated savings vs pointer model: {saved:,} B")
def _append_string_data_entry(lines: List[str], text: str) -> None:
@@ -917,6 +965,7 @@ def main(
string_keys,
unused_set,
data_sizes,
translations,
)
print()