Merge pull request #229 from jpirnay/feat-swedish-hyphenation

feat: Add Swedish hyphenation (upstream PR 1637 by steka)

lib/Epub/Epub/hyphenation/Hyphenator.cpp

@@ -8,15 +8,21 @@
 #include "LanguageHyphenator.h"
 #include "LanguageRegistry.h"
 
+struct Iso639Mapping {
+  const char* iso639_2;
+  const char* iso639_1;
+};
+static constexpr Iso639Mapping kIso639Mappings[] = {{"eng", "en"}, {"fra", "fr"}, {"fre", "fr"}, {"deu", "de"},
+                                                    {"ger", "de"}, {"rus", "ru"}, {"spa", "es"}, {"ita", "it"},
+                                                    {"ukr", "uk"}, {"swe", "sv"}, {"pol", "pl"}};
+
 const LanguageHyphenator* Hyphenator::cachedHyphenator_ = nullptr;
 
 namespace {
 
-// Maps a BCP-47 language tag to a language-specific hyphenator.
 const LanguageHyphenator* hyphenatorForLanguage(const std::string& langTag) {
   if (langTag.empty()) return nullptr;
 
-  // Extract primary subtag and normalize to lowercase (e.g., "en-US" -> "en").
   std::string primary;
   primary.reserve(langTag.size());
   for (char c : langTag) {
@@ -26,6 +32,12 @@ const LanguageHyphenator* hyphenatorForLanguage(const std::string& langTag) {
   }
   if (primary.empty()) return nullptr;
 
+  for (const auto& mapping : kIso639Mappings) {
+    if (primary == mapping.iso639_2) {
+      return getLanguageHyphenatorForPrimaryTag(mapping.iso639_1);
+    }
+  }
+
   return getLanguageHyphenatorForPrimaryTag(primary);
 }
 

lib/Epub/Epub/hyphenation/LanguageRegistry.cpp

@@ -11,6 +11,7 @@
 #include "generated/hyph-it.trie.h"
 #include "generated/hyph-pl.trie.h"
 #include "generated/hyph-ru.trie.h"
+#include "generated/hyph-sv.trie.h"
 #include "generated/hyph-uk.trie.h"
 
 namespace {
@@ -22,10 +23,11 @@ LanguageHyphenator germanHyphenator(de_patterns, isLatinLetter, toLowerLatin);
 LanguageHyphenator russianHyphenator(ru_patterns, isCyrillicLetter, toLowerCyrillic);
 LanguageHyphenator spanishHyphenator(es_patterns, isLatinLetter, toLowerLatin);
 LanguageHyphenator italianHyphenator(it_patterns, isLatinLetter, toLowerLatin);
+LanguageHyphenator swedishHyphenator(sv_patterns, isLatinLetter, toLowerLatin);
 LanguageHyphenator ukrainianHyphenator(uk_patterns, isCyrillicLetter, toLowerCyrillic);
 LanguageHyphenator polishHyphenator(pl_patterns, isLatinLetter, toLowerLatin);
 
-using EntryArray = std::array<LanguageEntry, 8>;
+using EntryArray = std::array<LanguageEntry, 9>;
 
 const EntryArray& entries() {
   static const EntryArray kEntries = {{{"english", "en", &englishHyphenator},
@@ -34,6 +36,7 @@ const EntryArray& entries() {
                                        {"russian", "ru", &russianHyphenator},
                                        {"spanish", "es", &spanishHyphenator},
                                        {"italian", "it", &italianHyphenator},
+                                       {"swedish", "sv", &swedishHyphenator},
                                        {"polish", "pl", &polishHyphenator},
                                        {"ukrainian", "uk", &ukrainianHyphenator}}};
   return kEntries;

scripts/generate_hyphenation_trie.py

@@ -73,7 +73,7 @@ constexpr SerializedHyphenationPatterns {patterns_symbol} = {{
     sizeof({data_symbol}),
 }};
 """
-    path.write_text(content)
+    path.write_text(content, encoding='utf-8')
 
 
 def main() -> None:

scripts/update_hypenation.sh

@@ -23,3 +23,4 @@ process es
 process ru
 process it
 process uk
+process sv

test/hyphenation_eval/HyphenationEvaluationTest.cpp

@@ -44,6 +44,7 @@ const std::vector<LanguageConfig> kSupportedLanguages = {
     {"russian", "test/hyphenation_eval/resources/russian_hyphenation_tests.txt", "ru"},
     {"spanish", "test/hyphenation_eval/resources/spanish_hyphenation_tests.txt", "es"},
     {"italian", "test/hyphenation_eval/resources/italian_hyphenation_tests.txt", "it"},
+    {"swedish", "test/hyphenation_eval/resources/swedish_hyphenation_tests.txt", "sv"},
 };
 
 std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) {