tobias/friendica
1
0
Fork 0
Code Pull requests Activity

New language detection

Browse source
This commit is contained in:
Michael 2020-10-03 15:42:21 +00:00
parent 852ee2f5c5
commit 2cd54320e4
3 changed files with 64 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
composer.json
View file

@ -40,7 +40,6 @@
"nikic/fast-route": "^1.3", "nikic/fast-route": "^1.3",
"paragonie/hidden-string": "^1.0", "paragonie/hidden-string": "^1.0",
"pear/console_table": "^1.3", "pear/console_table": "^1.3",
"pear/text_languagedetect": "1.*",
"pragmarx/google2fa": "^5.0", "pragmarx/google2fa": "^5.0",
"pragmarx/recovery": "^0.1.0", "pragmarx/recovery": "^0.1.0",
"psr/container": "^1.0", "psr/container": "^1.0",
@ -64,7 +63,8 @@
"npm-asset/perfect-scrollbar": "0.6.16", "npm-asset/perfect-scrollbar": "0.6.16",
"npm-asset/textcomplete": "^0.18.2", "npm-asset/textcomplete": "^0.18.2",
"npm-asset/typeahead.js": "^0.11.1", "npm-asset/typeahead.js": "^0.11.1",
"phpseclib/phpseclib": "^2.0" "phpseclib/phpseclib": "^2.0",
"patrickschur/language-detection": "^3.4"
}, },
"repositories": [ "repositories": [
{ {

105
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "ed9aa898eaf8a1f8a807f3be9eecc3d7", "content-hash": "2cd2011a7e93f6f64524d1663651c016",
"packages": [ "packages": [
{ {
"name": "asika/simple-console", "name": "asika/simple-console",
@ -460,7 +460,6 @@
"jsonld.php" "jsonld.php"
] ]
}, },
"notification-url": "https://packagist.org/downloads/",
"license": [ "license": [
"BSD-3-Clause" "BSD-3-Clause"
], ],
@ -478,11 +477,11 @@
"description": "A JSON-LD Processor and API implementation in PHP.", "description": "A JSON-LD Processor and API implementation in PHP.",
"homepage": "https://git.friendi.ca/friendica/php-json-ld", "homepage": "https://git.friendi.ca/friendica/php-json-ld",
"keywords": [ "keywords": [
"JSON",
"JSON-LD", "JSON-LD",
"Linked Data", "Linked Data",
"RDF", "RDF",
"Semantic Web", "Semantic Web",
"json",
"jsonld" "jsonld"
], ],
"time": "2018-10-08T20:41:00+00:00" "time": "2018-10-08T20:41:00+00:00"
@ -2379,6 +2378,52 @@
], ],
"time": "2020-03-20T21:48:09+00:00" "time": "2020-03-20T21:48:09+00:00"
}, },
{
"name": "patrickschur/language-detection",
"version": "v3.4.0",
"source": {
"type": "git",
"url": "https://github.com/patrickschur/language-detection.git",
"reference": "95b55109177d5c4bd6b1bec6e8835cd0df36ef5f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/patrickschur/language-detection/zipball/95b55109177d5c4bd6b1bec6e8835cd0df36ef5f",
"reference": "95b55109177d5c4bd6b1bec6e8835cd0df36ef5f",
"shasum": ""
},
"require": {
"ext-mbstring": "*",
"php": "^7"
},
"require-dev": {
"phpunit/phpunit": "^6"
},
"type": "library",
"autoload": {
"psr-4": {
"LanguageDetection\\": "src/LanguageDetection"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Patrick Schur",
"email": "patrick_schur@outlook.de"
}
],
"description": "A language detection library for PHP. Detects the language from a given text string.",
"homepage": "https://github.com/patrickschur/language-detection",
"keywords": [
"detect",
"detection",
"language"
],
"time": "2018-09-19T21:45:51+00:00"
},
{ {
"name": "pear/console_table", "name": "pear/console_table",
"version": "v1.3.1", "version": "v1.3.1",
@ -2434,50 +2479,6 @@
], ],
"time": "2018-01-25T20:47:17+00:00" "time": "2018-01-25T20:47:17+00:00"
}, },
{
"name": "pear/text_languagedetect",
"version": "v1.0.1",
"source": {
"type": "git",
"url": "https://github.com/pear/Text_LanguageDetect.git",
"reference": "9e253f26cef9a9066f53f200cc3e0684018cb5b5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pear/Text_LanguageDetect/zipball/9e253f26cef9a9066f53f200cc3e0684018cb5b5",
"reference": "9e253f26cef9a9066f53f200cc3e0684018cb5b5",
"shasum": ""
},
"require-dev": {
"phpunit/phpunit": "8.*|9.*"
},
"suggest": {
"ext-mbstring": "May require the mbstring PHP extension"
},
"type": "library",
"autoload": {
"psr-0": {
"Text": "./"
}
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
"./"
],
"license": [
"BSD-2-Clause"
],
"authors": [
{
"name": "Nicholas Pisarro",
"email": "taak@php.net",
"role": "Lead"
}
],
"description": "Identify human languages from text samples",
"homepage": "http://pear.php.net/package/Text_LanguageDetect",
"time": "2020-05-17T12:19:40+00:00"
},
{ {
"name": "phpseclib/phpseclib", "name": "phpseclib/phpseclib",
"version": "2.0.29", "version": "2.0.29",
@ -4450,7 +4451,7 @@
} }
], ],
"description": "Provides the functionality to compare PHP values for equality", "description": "Provides the functionality to compare PHP values for equality",
"homepage": "https://github.com/sebastianbergmann/comparator", "homepage": "http://www.github.com/sebastianbergmann/comparator",
"keywords": [ "keywords": [
"comparator", "comparator",
"compare", "compare",
@ -4552,7 +4553,7 @@
} }
], ],
"description": "Provides functionality to handle HHVM/PHP environments", "description": "Provides functionality to handle HHVM/PHP environments",
"homepage": "https://github.com/sebastianbergmann/environment", "homepage": "http://www.github.com/sebastianbergmann/environment",
"keywords": [ "keywords": [
"Xdebug", "Xdebug",
"environment", "environment",
@ -4620,7 +4621,7 @@
} }
], ],
"description": "Provides the functionality to export PHP variables for visualization", "description": "Provides the functionality to export PHP variables for visualization",
"homepage": "https://github.com/sebastianbergmann/exporter", "homepage": "http://www.github.com/sebastianbergmann/exporter",
"keywords": [ "keywords": [
"export", "export",
"exporter" "exporter"
@ -4672,7 +4673,7 @@
} }
], ],
"description": "Snapshotting of global state", "description": "Snapshotting of global state",
"homepage": "https://github.com/sebastianbergmann/global-state", "homepage": "http://www.github.com/sebastianbergmann/global-state",
"keywords": [ "keywords": [
"global state" "global state"
], ],
@ -4774,7 +4775,7 @@
} }
], ],
"description": "Provides functionality to recursively process PHP variables", "description": "Provides functionality to recursively process PHP variables",
"homepage": "https://github.com/sebastianbergmann/recursion-context", "homepage": "http://www.github.com/sebastianbergmann/recursion-context",
"time": "2016-11-19T07:33:16+00:00" "time": "2016-11-19T07:33:16+00:00"
}, },
{ {

15
src/Model/Item.php
View file

@ -43,8 +43,8 @@ use Friendica\Util\Map;
use Friendica\Util\Network; use Friendica\Util\Network;
use Friendica\Util\Strings; use Friendica\Util\Strings;
use Friendica\Worker\Delivery; use Friendica\Worker\Delivery;
use Text_LanguageDetect;
use Friendica\Repository\PermissionSet as RepPermissionSet; use Friendica\Repository\PermissionSet as RepPermissionSet;
use LanguageDetection\Language;
class Item class Item
{ {
@ -1699,10 +1699,10 @@ class Item
$item['plink'] = ($item['plink'] ?? '') ?: DI::baseUrl() . '/display/' . urlencode($item['guid']); $item['plink'] = ($item['plink'] ?? '') ?: DI::baseUrl() . '/display/' . urlencode($item['guid']);
$item['language'] = self::getLanguage($item);
$item['gravity'] = self::getGravity($item); $item['gravity'] = self::getGravity($item);
$item['language'] = self::getLanguage($item);
$default = ['url' => $item['author-link'], 'name' => $item['author-name'], $default = ['url' => $item['author-link'], 'name' => $item['author-name'],
'photo' => $item['author-avatar'], 'network' => $item['network']]; 'photo' => $item['author-avatar'], 'network' => $item['network']];
$item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, null, $default); $item['author-id'] = ($item['author-id'] ?? 0) ?: Contact::getIdForURL($item['author-link'], 0, null, $default);
@ -2472,11 +2472,14 @@ class Item
*/ */
private static function getLanguage(array $item) private static function getLanguage(array $item)
{ {
if (!in_array($item['gravity'], [GRAVITY_PARENT, GRAVITY_COMMENT])) {
return '';
}
$naked_body = BBCode::toPlaintext($item['body'], false); $naked_body = BBCode::toPlaintext($item['body'], false);
$ld = new Text_LanguageDetect(); $ld = new Language;
$ld->setNameMode(2); $languages = $ld->detect($naked_body)->limit(0, 3)->close();
$languages = $ld->detect($naked_body, 3);
if (is_array($languages)) { if (is_array($languages)) {
return json_encode($languages); return json_encode($languages);
} }