feat: Add swedish hyphenation (#1637)
## Summary * Add swedish hyphenation using scripts/update_hypenation.sh * Add hyphenation test data using the Swedish translation of Andy Weir's Project Hail Mary --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**NO**_
This commit is contained in:
@@ -22,10 +22,9 @@ struct Iso639Mapping {
|
|||||||
const char* iso639_2;
|
const char* iso639_2;
|
||||||
const char* iso639_1;
|
const char* iso639_1;
|
||||||
};
|
};
|
||||||
static constexpr Iso639Mapping kIso639Mappings[] = {
|
static constexpr Iso639Mapping kIso639Mappings[] = {{"eng", "en"}, {"fra", "fr"}, {"fre", "fr"}, {"deu", "de"},
|
||||||
{"eng", "en"}, {"fra", "fr"}, {"fre", "fr"}, {"deu", "de"}, {"ger", "de"},
|
{"ger", "de"}, {"rus", "ru"}, {"spa", "es"}, {"ita", "it"},
|
||||||
{"rus", "ru"}, {"spa", "es"}, {"ita", "it"}, {"ukr", "uk"},
|
{"ukr", "uk"}, {"swe", "sv"}};
|
||||||
};
|
|
||||||
|
|
||||||
// Maps a BCP-47 or ISO 639-2 language tag to a language-specific hyphenator.
|
// Maps a BCP-47 or ISO 639-2 language tag to a language-specific hyphenator.
|
||||||
const LanguageHyphenator* hyphenatorForLanguage(const std::string& langTag) {
|
const LanguageHyphenator* hyphenatorForLanguage(const std::string& langTag) {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
#include "generated/hyph-it.trie.h"
|
#include "generated/hyph-it.trie.h"
|
||||||
#include "generated/hyph-pl.trie.h"
|
#include "generated/hyph-pl.trie.h"
|
||||||
#include "generated/hyph-ru.trie.h"
|
#include "generated/hyph-ru.trie.h"
|
||||||
|
#include "generated/hyph-sv.trie.h"
|
||||||
#include "generated/hyph-uk.trie.h"
|
#include "generated/hyph-uk.trie.h"
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
@@ -22,10 +23,11 @@ LanguageHyphenator germanHyphenator(de_patterns, isLatinLetter, toLowerLatin);
|
|||||||
LanguageHyphenator russianHyphenator(ru_patterns, isCyrillicLetter, toLowerCyrillic);
|
LanguageHyphenator russianHyphenator(ru_patterns, isCyrillicLetter, toLowerCyrillic);
|
||||||
LanguageHyphenator spanishHyphenator(es_patterns, isLatinLetter, toLowerLatin);
|
LanguageHyphenator spanishHyphenator(es_patterns, isLatinLetter, toLowerLatin);
|
||||||
LanguageHyphenator italianHyphenator(it_patterns, isLatinLetter, toLowerLatin);
|
LanguageHyphenator italianHyphenator(it_patterns, isLatinLetter, toLowerLatin);
|
||||||
|
LanguageHyphenator swedishHyphenator(sv_patterns, isLatinLetter, toLowerLatin);
|
||||||
LanguageHyphenator ukrainianHyphenator(uk_patterns, isCyrillicLetter, toLowerCyrillic);
|
LanguageHyphenator ukrainianHyphenator(uk_patterns, isCyrillicLetter, toLowerCyrillic);
|
||||||
LanguageHyphenator polishHyphenator(pl_patterns, isLatinLetter, toLowerLatin);
|
LanguageHyphenator polishHyphenator(pl_patterns, isLatinLetter, toLowerLatin);
|
||||||
|
|
||||||
using EntryArray = std::array<LanguageEntry, 8>;
|
using EntryArray = std::array<LanguageEntry, 9>;
|
||||||
|
|
||||||
const EntryArray& entries() {
|
const EntryArray& entries() {
|
||||||
static const EntryArray kEntries = {{{"english", "en", &englishHyphenator},
|
static const EntryArray kEntries = {{{"english", "en", &englishHyphenator},
|
||||||
@@ -35,6 +37,7 @@ const EntryArray& entries() {
|
|||||||
{"spanish", "es", &spanishHyphenator},
|
{"spanish", "es", &spanishHyphenator},
|
||||||
{"italian", "it", &italianHyphenator},
|
{"italian", "it", &italianHyphenator},
|
||||||
{"polish", "pl", &polishHyphenator},
|
{"polish", "pl", &polishHyphenator},
|
||||||
|
{"swedish", "sv", &swedishHyphenator},
|
||||||
{"ukrainian", "uk", &ukrainianHyphenator}}};
|
{"ukrainian", "uk", &ukrainianHyphenator}}};
|
||||||
return kEntries;
|
return kEntries;
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -24,3 +24,4 @@ process ru
|
|||||||
process it
|
process it
|
||||||
process uk
|
process uk
|
||||||
process pl
|
process pl
|
||||||
|
process sv
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ const std::vector<LanguageConfig> kSupportedLanguages = {
|
|||||||
{"spanish", "test/hyphenation_eval/resources/spanish_hyphenation_tests.txt", "es"},
|
{"spanish", "test/hyphenation_eval/resources/spanish_hyphenation_tests.txt", "es"},
|
||||||
{"italian", "test/hyphenation_eval/resources/italian_hyphenation_tests.txt", "it"},
|
{"italian", "test/hyphenation_eval/resources/italian_hyphenation_tests.txt", "it"},
|
||||||
{"polish", "test/hyphenation_eval/resources/polish_hyphenation_tests.txt", "pl"},
|
{"polish", "test/hyphenation_eval/resources/polish_hyphenation_tests.txt", "pl"},
|
||||||
|
{"swedish", "test/hyphenation_eval/resources/swedish_hyphenation_tests.txt", "sv"},
|
||||||
};
|
};
|
||||||
|
|
||||||
std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) {
|
std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user