Merge pull request #11884 from annando/language

Improved language detection

src/Model/Item.php

@@ -919,8 +919,6 @@ class Item
 
 		$item['gravity'] = self::getGravity($item);
 
-		$item['language'] = self::getLanguage($item);
-
 		$default = ['url' => $item['author-link'], 'name' => $item['author-name'],
 			'photo' => $item['author-avatar'], 'network' => $item['network']];
 		$item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, null, $default);
@@ -1108,6 +1106,8 @@ class Item
 		// Check for hashtags in the body and repair or add hashtag links
 		$item['body'] = self::setHashtags($item['body']);
 
+		$item['language'] = self::getLanguage($item);
+
 		$notify_type = Delivery::POST;
 
 		// Filling item related side tables
@@ -1869,6 +1869,8 @@ class Item
 			return '';
 		}
 
+		$naked_body = self::getDominantLanguage($naked_body);
+
 		$availableLanguages = DI::l10n()->getAvailableLanguages();
 		// See https://github.com/friendica/friendica/issues/10511
 		// Persian is manually added to language detection until a persian translation is provided for the interface, at
@@ -1884,6 +1886,33 @@ class Item
 		return '';
 	}
 
+	/**
+	 * Check if latin or non latin are dominant in the body and only return the dominant one
+	 *
+	 * @param string $body
+	 * @return string
+	 */
+	private static function getDominantLanguage(string $body): string
+	{
+		$latin = '';
+		$non_latin = '';
+		for ($i = 0; $i < mb_strlen($body); $i++) { 
+			$character = mb_substr($body, $i, 1);
+			$ord = mb_ord($character);
+
+			// We add the most common characters to both strings.
+			if (($ord <= 64) || ($ord >= 91 && $ord <= 96) || ($ord >= 123 && $ord <= 191) || in_array($ord, [215, 247]) || ($ord >= 697 && $ord <= 735) || ($ord > 65535)) {
+				$latin .= $character;
+				$non_latin .= $character;
+			} elseif ($ord < 768) {
+				$latin .= $character;
+			} else {
+				$non_latin .= $character;
+			}
+		}
+		return (mb_strlen($latin) > mb_strlen($non_latin)) ? $latin : $non_latin;
+	}
+
 	public static function getLanguageMessage(array $item): string
 	{
 		$iso639 = new \Matriphe\ISO639\ISO639;

view/lang/C/messages.po

@@ -1411,7 +1411,7 @@ msgstr ""
 msgid "Friend Suggestions"
 msgstr ""
 
-#: mod/tagger.php:78 src/Content/Item.php:297 src/Model/Item.php:2785
+#: mod/tagger.php:78 src/Content/Item.php:297 src/Model/Item.php:2814
 msgid "photo"
 msgstr ""
 
@@ -2292,7 +2292,7 @@ msgstr ""
 msgid "show more"
 msgstr ""
 
-#: src/Content/Item.php:288 src/Model/Item.php:2783
+#: src/Content/Item.php:288 src/Model/Item.php:2812
 msgid "event"
 msgstr ""
 
@@ -2639,8 +2639,8 @@ msgid ""
 "<a href=\"%1$s\" target=\"_blank\" rel=\"noopener noreferrer\">%2$s</a> %3$s"
 msgstr ""
 
-#: src/Content/Text/BBCode.php:1213 src/Model/Item.php:3359
+#: src/Content/Text/BBCode.php:1213 src/Model/Item.php:3388
-#: src/Model/Item.php:3365 src/Model/Item.php:3366
+#: src/Model/Item.php:3394 src/Model/Item.php:3395
 msgid "Link to source"
 msgstr ""
 
@@ -3782,66 +3782,66 @@ msgstr ""
 msgid "Edit groups"
 msgstr ""
 
-#: src/Model/Item.php:1895
+#: src/Model/Item.php:1924
 #, php-format
 msgid "Detected languages in this post:\\n%s"
 msgstr ""
 
-#: src/Model/Item.php:2787
+#: src/Model/Item.php:2816
 msgid "activity"
 msgstr ""
 
-#: src/Model/Item.php:2789
+#: src/Model/Item.php:2818
 msgid "comment"
 msgstr ""
 
-#: src/Model/Item.php:2792
+#: src/Model/Item.php:2821
 msgid "post"
 msgstr ""
 
-#: src/Model/Item.php:2908
+#: src/Model/Item.php:2937
 #, php-format
 msgid "Content warning: %s"
 msgstr ""
 
-#: src/Model/Item.php:3271
+#: src/Model/Item.php:3300
 msgid "bytes"
 msgstr ""
 
-#: src/Model/Item.php:3302
+#: src/Model/Item.php:3331
 #, php-format
 msgid "%2$s (%3$d%%, %1$d vote)"
 msgid_plural "%2$s (%3$d%%, %1$d votes)"
 msgstr[0] ""
 msgstr[1] ""
 
-#: src/Model/Item.php:3304
+#: src/Model/Item.php:3333
 #, php-format
 msgid "%2$s (%1$d vote)"
 msgid_plural "%2$s (%1$d votes)"
 msgstr[0] ""
 msgstr[1] ""
 
-#: src/Model/Item.php:3309
+#: src/Model/Item.php:3338
 #, php-format
 msgid "%d voter. Poll end: %s"
 msgid_plural "%d voters. Poll end: %s"
 msgstr[0] ""
 msgstr[1] ""
 
-#: src/Model/Item.php:3311
+#: src/Model/Item.php:3340
 #, php-format
 msgid "%d voter."
 msgid_plural "%d voters."
 msgstr[0] ""
 msgstr[1] ""
 
-#: src/Model/Item.php:3313
+#: src/Model/Item.php:3342
 #, php-format
 msgid "Poll end: %s"
 msgstr ""
 
-#: src/Model/Item.php:3347 src/Model/Item.php:3348
+#: src/Model/Item.php:3376 src/Model/Item.php:3377
 msgid "View on separate page"
 msgstr ""