feat: Add swedish hyphenation (#1637)

## Summary

* Add swedish hyphenation using scripts/update_hypenation.sh
* Add hyphenation test data using the Swedish translation of Andy Weir's
Project Hail Mary

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.

Did you use AI tools to help write this code? _**NO**_
This commit is contained in:
Stefan Blixten Karlsson
2026-05-15 21:27:52 -05:00
committed by GitHub
parent 2f342508bc
commit 77afea4d95
6 changed files with 6513 additions and 6 deletions
+3 -4
View File
@@ -22,10 +22,9 @@ struct Iso639Mapping {
const char* iso639_2;
const char* iso639_1;
};
static constexpr Iso639Mapping kIso639Mappings[] = {
{"eng", "en"}, {"fra", "fr"}, {"fre", "fr"}, {"deu", "de"}, {"ger", "de"},
{"rus", "ru"}, {"spa", "es"}, {"ita", "it"}, {"ukr", "uk"},
};
static constexpr Iso639Mapping kIso639Mappings[] = {{"eng", "en"}, {"fra", "fr"}, {"fre", "fr"}, {"deu", "de"},
{"ger", "de"}, {"rus", "ru"}, {"spa", "es"}, {"ita", "it"},
{"ukr", "uk"}, {"swe", "sv"}};
// Maps a BCP-47 or ISO 639-2 language tag to a language-specific hyphenator.
const LanguageHyphenator* hyphenatorForLanguage(const std::string& langTag) {
@@ -11,6 +11,7 @@
#include "generated/hyph-it.trie.h"
#include "generated/hyph-pl.trie.h"
#include "generated/hyph-ru.trie.h"
#include "generated/hyph-sv.trie.h"
#include "generated/hyph-uk.trie.h"
namespace {
@@ -22,10 +23,11 @@ LanguageHyphenator germanHyphenator(de_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator russianHyphenator(ru_patterns, isCyrillicLetter, toLowerCyrillic);
LanguageHyphenator spanishHyphenator(es_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator italianHyphenator(it_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator swedishHyphenator(sv_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator ukrainianHyphenator(uk_patterns, isCyrillicLetter, toLowerCyrillic);
LanguageHyphenator polishHyphenator(pl_patterns, isLatinLetter, toLowerLatin);
using EntryArray = std::array<LanguageEntry, 8>;
using EntryArray = std::array<LanguageEntry, 9>;
const EntryArray& entries() {
static const EntryArray kEntries = {{{"english", "en", &englishHyphenator},
@@ -35,6 +37,7 @@ const EntryArray& entries() {
{"spanish", "es", &spanishHyphenator},
{"italian", "it", &italianHyphenator},
{"polish", "pl", &polishHyphenator},
{"swedish", "sv", &swedishHyphenator},
{"ukrainian", "uk", &ukrainianHyphenator}}};
return kEntries;
}
File diff suppressed because it is too large Load Diff
+1
View File
@@ -24,3 +24,4 @@ process ru
process it
process uk
process pl
process sv
@@ -45,6 +45,7 @@ const std::vector<LanguageConfig> kSupportedLanguages = {
{"spanish", "test/hyphenation_eval/resources/spanish_hyphenation_tests.txt", "es"},
{"italian", "test/hyphenation_eval/resources/italian_hyphenation_tests.txt", "it"},
{"polish", "test/hyphenation_eval/resources/polish_hyphenation_tests.txt", "pl"},
{"swedish", "test/hyphenation_eval/resources/swedish_hyphenation_tests.txt", "sv"},
};
std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) {
@@ -388,4 +389,4 @@ int main(int argc, char* argv[]) {
}
return 0;
}
}
File diff suppressed because it is too large Load Diff