Merge branch 'refactor-i18n_offset-table' of https://github.com/jpirnay/crosspoint-reader into mybuild

scripts/gen_i18n.py

@@ -146,6 +146,29 @@ def load_translations(
     if english_file is None:
         raise ValueError("No YAML file with _language_code: EN found")
 
+    duplicate_orders: Dict[str, List[str]] = {}
+    order_to_files: Dict[str, List[str]] = {}
+    for fname, data in parsed.items():
+        order = data.get("_order")
+        if not order:
+            continue
+        order_to_files.setdefault(order, []).append(fname)
+
+    for order, files in order_to_files.items():
+        if len(files) > 1:
+            duplicate_orders[order] = sorted(files)
+
+    if duplicate_orders:
+        duplicate_messages = [
+            f"_order {order}: {', '.join(files)}"
+            for order, files in sorted(
+                duplicate_orders.items(), key=lambda item: int(item[0])
+            )
+        ]
+        raise ValueError(
+            "Duplicate _order values found:\n  " + "\n  ".join(duplicate_messages) + "\nEach _order value must be unique to ensure a deterministic language order."
+        )
+
     # Order: English first, then by _order metadata (falls back to filename)
     def sort_key(fname: str) -> Tuple[int, int, str]:
         """English always first (0), then by _order, then by filename."""
@@ -266,57 +289,13 @@ def report_unused_keys(
     used_keys: Set[str],
 ) -> List[str]:
     """Return a sorted list of keys from *string_keys* absent in *used_keys*."""
-    return [k for k in string_keys if k not in used_keys]
+    return [k for k in sorted(string_keys) if k not in used_keys]
 
 
 # ---------------------------------------------------------------------------
 # C++ string escaping
 # ---------------------------------------------------------------------------
 
-LANG_ABBREVIATIONS = {
-    "english": "EN",
-    "español": "ES",
-    "espanol": "ES",
-    "italiano": "IT",
-    "svenska": "SV",
-    "français": "FR",
-    "francais": "FR",
-    "deutsch": "DE",
-    "german": "DE",
-    "polski": "PL",
-    "português": "PT",
-    "portugues": "PT",
-    "português (brasil)": "PO",
-    "中文": "ZH",
-    "chinese": "ZH",
-    "日本語": "JA",
-    "japanese": "JA",
-    "한국어": "KO",
-    "korean": "KO",
-    "русский": "RU",
-    "russian": "RU",
-    "العربية": "AR",
-    "arabic": "AR",
-    "עברית": "HE",
-    "hebrew": "HE",
-    "فارسی": "FA",
-    "persian": "FA",
-    "čeština": "CS",
-    "türkçe": "TR",
-    "turkish": "TR",
-    "Қазақша": "KK",
-    "kazakh": "KK",
-}
-
-
-def get_lang_abbreviation(lang_code: str, lang_name: str) -> str:
-    """Return a 2-letter abbreviation for a language."""
-    lower = lang_name.lower()
-    if lower in LANG_ABBREVIATIONS:
-        return LANG_ABBREVIATIONS[lower]
-    return lang_code[:2].upper()
-
-
 def escape_cpp_string(s: str) -> List[str]:
     r"""
     Convert *s* into one or more C++ string literal segments.
@@ -471,10 +450,9 @@ def generate_keys_header(
         "namespace i18n_strings {",
     ]
 
-    for code, name in zip(languages, language_names):
+    for code in languages:
-        abbrev = get_lang_abbreviation(code, name)
+        lines.append(f"extern const char STRINGS_{code}_DATA[];")
-        lines.append(f"extern const char STRINGS_{abbrev}_DATA[];")
+        lines.append(f"extern const uint16_t OFFSETS_{code}[];")
-        lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];")
 
     lines.append("}  // namespace i18n_strings")
     lines.append("")
@@ -518,16 +496,15 @@ def generate_keys_header(
     lines.append("// Helper function to get string data for a language")
     lines.append("inline LangStrings getLanguageStrings(Language lang) {")
     lines.append("  switch (lang) {")
-    for code, name in zip(languages, language_names):
+    for code in languages:
-        abbrev = get_lang_abbreviation(code, name)
         lines.append(f"    case Language::{code}:")
         lines.append(
-            f"      return {{i18n_strings::STRINGS_{abbrev}_DATA, i18n_strings::OFFSETS_{abbrev}}};"
+            f"      return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};"
         )
-    first_abbrev = get_lang_abbreviation(languages[0], language_names[0])
+    first_code = languages[0]
     lines.append("    default:")
     lines.append(
-        f"      return {{i18n_strings::STRINGS_{first_abbrev}_DATA, i18n_strings::OFFSETS_{first_abbrev}}};"
+        f"      return {{i18n_strings::STRINGS_{first_code}_DATA, i18n_strings::OFFSETS_{first_code}}};"
     )
     lines.append("  }")
     lines.append("}")
@@ -583,10 +560,9 @@ def generate_strings_header(
         "",
     ]
 
-    for code, name in zip(languages, language_names):
+    for code in languages:
-        abbrev = get_lang_abbreviation(code, name)
+        lines.append(f"extern const char STRINGS_{code}_DATA[];")
-        lines.append(f"extern const char STRINGS_{abbrev}_DATA[];")
+        lines.append(f"extern const uint16_t OFFSETS_{code}[];")
-        lines.append(f"extern const uint16_t OFFSETS_{abbrev}[];")
 
     lines.append("")
     lines.append("}  // namespace i18n_strings")
@@ -631,8 +607,7 @@ def generate_strings_cpp(
     lines.append("namespace i18n_strings {")
     lines.append("")
 
-    for lang_idx, (code, name) in enumerate(zip(languages, language_names)):
+    for lang_idx, code in enumerate(languages):
-        abbrev = get_lang_abbreviation(code, name)
         lang_strings = [translations[key][lang_idx] for key in string_keys]
 
         # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator)
@@ -650,7 +625,7 @@ def generate_strings_cpp(
         # Flat string data blob — all strings concatenated with \0 separators.
         # clang-format off/on avoids reformatting of the adjacent string literals.
         lines.append("// clang-format off")
-        lines.append(f"const char STRINGS_{abbrev}_DATA[] =")
+        lines.append(f"const char STRINGS_{code}_DATA[] =")
         for text in lang_strings:
             _append_string_data_entry(lines, text)
         lines.append(";")
@@ -658,7 +633,7 @@ def generate_strings_cpp(
         lines.append("")
 
         # Offset table — one uint16_t per StrId
-        lines.append(f"const uint16_t OFFSETS_{abbrev}[] = {{")
+        lines.append(f"const uint16_t OFFSETS_{code}[] = {{")
         chunk_size = 12
         for i in range(0, len(offsets), chunk_size):
             chunk = offsets[i : i + chunk_size]
@@ -671,14 +646,13 @@ def generate_strings_cpp(
 
     # Compile-time size checks
     lines.append("// Compile-time validation of array sizes")
-    for code, name in zip(languages, language_names):
+    for code in languages:
-        abbrev = get_lang_abbreviation(code, name)
         lines.append(
-            f"static_assert(sizeof(i18n_strings::OFFSETS_{abbrev}) "
+            f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) "
-            f"/ sizeof(i18n_strings::OFFSETS_{abbrev}[0]) =="
+            f"/ sizeof(i18n_strings::OFFSETS_{code}[0]) =="
         )
         lines.append("                  static_cast<size_t>(StrId::_COUNT),")
-        lines.append(f'              "OFFSETS_{abbrev} size mismatch");')
+        lines.append(f'              "OFFSETS_{code} size mismatch");')
 
     _write_file(output_path, lines, verbose)
 
@@ -846,6 +820,17 @@ def main(
             used_keys = set(string_keys)
             unused_set = set()
 
+        # --- Missing-string detection (used in code but absent from English) ---
+        missing_keys = sorted(used_keys - set(string_keys))
+        if missing_keys:
+            print(
+                f"\n  CRITICAL: {len(missing_keys)} string(s) used in source but missing from english.yaml:"
+            )
+            for key in missing_keys:
+                print(f"    - {key}")
+            print()
+            sys.exit(1)
+
         # Compute per-language data blob sizes:
         # sum of UTF-8 byte length + 1 (null terminator) per string
         data_sizes = [