friendica/friendica-addons
8
3
Fork 14
Code Issues 3 Pull requests 1 Releases 16 Wiki Activity

CLD2: Use ISO-639-1 for the language detection #1433

Merged
MrPetovan merged 1 commit from heluecht/friendica-addons:ISO-639-1 into 2023.09-rc 2023-11-03 19:07:05 +01:00
Conversation 0 Commits 1 Files changed 1 +5 -30
1 changed files with 5 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
cld/cld.php
View file

@ -35,35 +35,10 @@ function cld_detect_languages(array &$data)
$original = ''; $original = '';
} }
$detected = $result['language_code']; $detected = DI::l10n()->toISO6391($result['language_code']);
if ($detected == 'pt') {
$detected = 'pt-PT';
} elseif ($detected == 'az') {
$detected = 'az-Latn';
} elseif ($detected == 'bs') {
$detected = 'bs-Latn';
} elseif ($detected == 'el') {
$detected = 'el-monoton';
} elseif ($detected == 'ht') {
$detected = 'fr';
} elseif ($detected == 'iw') {
$detected = 'he';
} elseif ($detected == 'jw') {
$detected = 'jv';
} elseif ($detected == 'ms') {
$detected = 'ms-Latn';
} elseif ($detected == 'no') {
$detected = 'nb';
} elseif ($detected == 'sr') {
$detected = 'sr-Cyrl';
} elseif ($detected == 'zh') {
$detected = 'zh-Hans';
} elseif ($detected == 'zh-Hant') {
$detected = 'zh-hant';
}
// languages that aren't supported via the base language detection // languages that aren't supported via the base language detection or tend to false detections
if (in_array($detected, ['ceb', 'hmn', 'ht', 'kk', 'ky', 'mg', 'mk', 'ml', 'ny', 'or', 'pa', 'rw', 'su', 'st', 'tg', 'ts', 'xx-Qaai'])) { if ((strlen($detected) == 3) || in_array($detected, ['ht', 'kk', 'ku', 'ky', 'lg', 'mg', 'mk', 'mt', 'ny', 'rw', 'st', 'su', 'tg', 'ts', 'xx'])) {
return; return;
} }
@ -75,7 +50,7 @@ function cld_detect_languages(array &$data)
return; return;
} }
$available = array_keys(DI::l10n()->convertForLanguageDetection(DI::l10n()->getAvailableLanguages(true))); $available = array_keys(DI::l10n()->getLanguageCodes());
if (!in_array($detected, $available)) { if (!in_array($detected, $available)) {
Logger::debug('Unsupported language', ['uri-id' => $data['uri-id'], 'original' => $original, 'detected' => $detected, 'name' => $result['language_name'], 'probability' => $result['language_probability'], 'text' => $data['text']]); Logger::debug('Unsupported language', ['uri-id' => $data['uri-id'], 'original' => $original, 'detected' => $detected, 'name' => $result['language_name'], 'probability' => $result['language_probability'], 'text' => $data['text']]);