Tupambae_Org/friendica_2021-01
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #9332 from annando/relayed-dfrn-dspr

Browse source 
New "relay" class / check of incoming posts from DFRN and Diaspora
This commit is contained in:
Philipp 2020-10-02 15:06:13 +02:00 committed by GitHub
commit 78121afcb0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 219 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
src/Protocol/ActivityPub/Processor.php
View file

@ -35,15 +35,14 @@ use Friendica\Model\Event;
use Friendica\Model\Item; use Friendica\Model\Item;
use Friendica\Model\ItemURI; use Friendica\Model\ItemURI;
use Friendica\Model\Mail; use Friendica\Model\Mail;
use Friendica\Model\Search;
use Friendica\Model\Tag; use Friendica\Model\Tag;
use Friendica\Model\User; use Friendica\Model\User;
use Friendica\Protocol\Activity; use Friendica\Protocol\Activity;
use Friendica\Protocol\ActivityPub; use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\Relay;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\JsonLD; use Friendica\Util\JsonLD;
use Friendica\Util\Strings; use Friendica\Util\Strings;
use Text_LanguageDetect;
/** /**
* ActivityPub Processor Protocol class * ActivityPub Processor Protocol class
@ -801,25 +800,16 @@ class Processor
return true; return true;
} }
$config = DI::config();
$subscribe = $config->get('system', 'relay_subscribe', false);
if ($subscribe) {
$scope = $config->get('system', 'relay_scope', SR_SCOPE_ALL);
} else {
$scope = SR_SCOPE_NONE;
}
$replyto = JsonLD::fetchElement($activity['as:object'], 'as:inReplyTo', '@id'); $replyto = JsonLD::fetchElement($activity['as:object'], 'as:inReplyTo', '@id');
if (Item::exists(['uri' => $replyto])) { if (Item::exists(['uri' => $replyto])) {
Logger::info('Post is a reply to an existing post - accepted', ['id' => $id, 'replyto' => $replyto]); Logger::info('Post is a reply to an existing post - accepted', ['id' => $id, 'replyto' => $replyto]);
return true; return true;
} }
if ($scope == SR_SCOPE_NONE) { $attributed_to = JsonLD::fetchElement($activity['as:object'], 'as:attributedTo', '@id');
Logger::info('Server does not accept relay posts - rejected', ['id' => $id]); $authorid = Contact::getIdForURL($attributed_to);
return false;
} $body = HTML::toBBCode(JsonLD::fetchElement($activity['as:object'], 'as:content', '@value'));
$messageTags = []; $messageTags = [];
$tags = Receiver::processTags(JsonLD::fetchElementArray($activity['as:object'], 'as:tag') ?? []); $tags = Receiver::processTags(JsonLD::fetchElementArray($activity['as:object'], 'as:tag') ?? []);
@ -832,61 +822,7 @@ class Processor
} }
} }
$systemTags = []; return Relay::isSolicitedPost($messageTags, $body, $authorid, $id, Protocol::ACTIVITYPUB);
$userTags = [];
$denyTags = [];
if ($scope == SR_SCOPE_TAGS) {
$server_tags = $config->get('system', 'relay_server_tags');
$tagitems = explode(',', mb_strtolower($server_tags));
foreach ($tagitems AS $tag) {
$systemTags[] = trim($tag, '# ');
}
if ($config->get('system', 'relay_user_tags')) {
$userTags = Search::getUserTags();
}
}
$tagList = array_unique(array_merge($systemTags, $userTags));
$deny_tags = $config->get('system', 'relay_deny_tags');
$tagitems = explode(',', mb_strtolower($deny_tags));
foreach ($tagitems AS $tag) {
$tag = trim($tag, '# ');
$denyTags[] = $tag;
}
if (!empty($tagList) || !empty($denyTags)) {
$content = mb_strtolower(BBCode::toPlaintext(HTML::toBBCode(JsonLD::fetchElement($activity['as:object'], 'as:content', '@value')), false));
foreach ($messageTags as $tag) {
if (in_array($tag, $denyTags)) {
Logger::info('Unwanted hashtag found - rejected', ['id' => $id, 'hashtag' => $tag]);
return false;
}
if (in_array($tag, $tagList)) {
Logger::info('Subscribed hashtag found - accepted', ['id' => $id, 'hashtag' => $tag]);
return true;
}
// We check with "strpos" for performance issues. Only when this is true, the regular expression check is used
// RegExp is taken from here: https://medium.com/@shiba1014/regex-word-boundaries-with-unicode-207794f6e7ed
if ((strpos($content, $tag) !== false) && preg_match('/(?<=[\s,.:;"\']|^)' . preg_quote($tag, '/') . '(?=[\s,.:;"\']|$)/', $content)) {
Logger::info('Subscribed hashtag found in content - accepted', ['id' => $id, 'hashtag' => $tag]);
return true;
}
}
}
if ($scope == SR_SCOPE_ALL) {
Logger::info('Server accept all posts - accepted', ['id' => $id]);
return true;
}
Logger::info('No matching hashtags found - rejected', ['id' => $id]);
return false;
} }
/** /**

27
src/Protocol/DFRN.php
View file

@ -2261,6 +2261,25 @@ class DFRN
} }
} }
/**
* Checks if an incoming message is wanted
*
* @param array $item
* @return boolean Is the message wanted?
*/
private static function isSolicitedMessage(array $item)
{
if (DBA::exists('contact', ["`nurl` = ? AND `uid` != ? AND `rel` IN (?, ?)",
Strings::normaliseLink($item["author-link"]), 0, Contact::FRIEND, Contact::SHARING])) {
Logger::info('Author has got followers - accepted', ['uri' => $item['uri'], 'author' => $item["author-link"]]);
return true;
}
$taglist = Tag::getByURIId($item['uri-id'], [Tag::HASHTAG]);
$tags = array_column($taglist, 'name');
return Relay::isSolicitedPost($tags, $item['body'], $item['author-id'], $item['uri'], Protocol::DFRN);
}
/** /**
* Processes the entry elements which contain the items and comments * Processes the entry elements which contain the items and comments
* *
@ -2450,6 +2469,14 @@ class DFRN
} }
} }
// Check if the message is wanted
if (($importer["importer_uid"] == 0) && ($item['uri'] == $item['parent-uri'])) {
if (!self::isSolicitedMessage($item)) {
DBA::delete('item-uri', ['uri' => $item['uri']]);
return 403;
}
}
// Get the type of the item (Top level post, reply or remote reply) // Get the type of the item (Top level post, reply or remote reply)
$entrytype = self::getEntryType($importer, $item); $entrytype = self::getEntryType($importer, $item);

61
src/Protocol/Diaspora.php
View file

@ -640,12 +640,13 @@ class Diaspora
* Dispatches public messages and find the fitting receivers * Dispatches public messages and find the fitting receivers
* *
* @param array $msg The post that will be dispatched * @param array $msg The post that will be dispatched
* @param bool $fetched The message had been fetched (default "false")
* *
* @return int The message id of the generated message, "true" or "false" if there was an error * @return int The message id of the generated message, "true" or "false" if there was an error
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException * @throws \ImagickException
*/ */
public static function dispatchPublic($msg) public static function dispatchPublic($msg, bool $fetched = false)
{ {
$enabled = intval(DI::config()->get("system", "diaspora_enabled")); $enabled = intval(DI::config()->get("system", "diaspora_enabled"));
if (!$enabled) { if (!$enabled) {
@ -659,7 +660,7 @@ class Diaspora
} }
$importer = ["uid" => 0, "page-flags" => User::PAGE_FLAGS_FREELOVE]; $importer = ["uid" => 0, "page-flags" => User::PAGE_FLAGS_FREELOVE];
$success = self::dispatch($importer, $msg, $fields); $success = self::dispatch($importer, $msg, $fields, $fetched);
return $success; return $success;
} }
@ -670,12 +671,13 @@ class Diaspora
* @param array $importer Array of the importer user * @param array $importer Array of the importer user
* @param array $msg The post that will be dispatched * @param array $msg The post that will be dispatched
* @param SimpleXMLElement $fields SimpleXML object that contains the message * @param SimpleXMLElement $fields SimpleXML object that contains the message
* @param bool $fetched The message had been fetched (default "false")
* *
* @return int The message id of the generated message, "true" or "false" if there was an error * @return int The message id of the generated message, "true" or "false" if there was an error
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException * @throws \ImagickException
*/ */
public static function dispatch(array $importer, $msg, SimpleXMLElement $fields = null) public static function dispatch(array $importer, $msg, SimpleXMLElement $fields = null, bool $fetched = false)
{ {
// The sender is the handle of the contact that sent the message. // The sender is the handle of the contact that sent the message.
// This will often be different with relayed messages (for example "like" and "comment") // This will often be different with relayed messages (for example "like" and "comment")
@ -708,7 +710,7 @@ class Diaspora
return self::receiveAccountDeletion($fields); return self::receiveAccountDeletion($fields);
case "comment": case "comment":
return self::receiveComment($importer, $sender, $fields, $msg["message"]); return self::receiveComment($importer, $sender, $fields, $msg["message"], $fetched);
case "contact": case "contact":
if (!$private) { if (!$private) {
@ -761,7 +763,7 @@ class Diaspora
return self::receiveRetraction($importer, $sender, $fields); return self::receiveRetraction($importer, $sender, $fields);
case "status_message": case "status_message":
return self::receiveStatusMessage($importer, $fields, $msg["message"]); return self::receiveStatusMessage($importer, $fields, $msg["message"], $fetched);
default: default:
Logger::log("Unknown message type ".$type); Logger::log("Unknown message type ".$type);
@ -1238,7 +1240,7 @@ class Diaspora
Logger::log("Successfully fetched item ".$guid." from ".$server, Logger::DEBUG); Logger::log("Successfully fetched item ".$guid." from ".$server, Logger::DEBUG);
// Now call the dispatcher // Now call the dispatcher
return self::dispatchPublic($msg); return self::dispatchPublic($msg, true);
} }
/** /**
@ -1674,12 +1676,13 @@ class Diaspora
* @param string $sender The sender of the message * @param string $sender The sender of the message
* @param object $data The message object * @param object $data The message object
* @param string $xml The original XML of the message * @param string $xml The original XML of the message
* @param bool $fetched The message had been fetched and not pushed
* *
* @return int The message id of the generated comment or "false" if there was an error * @return int The message id of the generated comment or "false" if there was an error
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException * @throws \ImagickException
*/ */
private static function receiveComment(array $importer, $sender, $data, $xml) private static function receiveComment(array $importer, $sender, $data, $xml, bool $fetched)
{ {
$author = Strings::escapeTags(XML::unescape($data->author)); $author = Strings::escapeTags(XML::unescape($data->author));
$guid = Strings::escapeTags(XML::unescape($data->guid)); $guid = Strings::escapeTags(XML::unescape($data->guid));
@ -1736,7 +1739,13 @@ class Diaspora
$datarray["owner-id"] = Contact::getIdForURL($contact["url"], 0); $datarray["owner-id"] = Contact::getIdForURL($contact["url"], 0);
// Will be overwritten for sharing accounts in Item::insert // Will be overwritten for sharing accounts in Item::insert
$datarray['post-type'] = ($datarray["uid"] == 0) ? Item::PT_GLOBAL : Item::PT_COMMENT; if ($fetched) {
$datarray["post-type"] = Item::PT_FETCHED;
} elseif ($datarray["uid"] == 0) {
$datarray["post-type"] = Item::PT_GLOBAL;
} else {
$datarray["post-type"] = Item::PT_COMMENT;
}
$datarray["guid"] = $guid; $datarray["guid"] = $guid;
$datarray["uri"] = self::getUriFromGuid($author, $guid); $datarray["uri"] = self::getUriFromGuid($author, $guid);
@ -2778,18 +2787,41 @@ class Diaspora
return true; return true;
} }
/**
* Checks if an incoming message is wanted
*
* @param string $url
* @param integer $uriid
* @param string $author
* @param string $body
* @return boolean Is the message wanted?
*/
private static function isSolicitedMessage(string $url, int $uriid, string $author, string $body)
{
$contact = Contact::getByURL($author);
if (DBA::exists('contact', ["`nurl` = ? AND `uid` != ? AND `rel` IN (?, ?)",
$contact['nurl'], 0, Contact::FRIEND, Contact::SHARING])) {
Logger::info('Author has got followers - accepted', ['url' => $url, 'author' => $author]);
return true;
}
$taglist = Tag::getByURIId($uriid, [Tag::HASHTAG]);
$tags = array_column($taglist, 'name');
return Relay::isSolicitedPost($tags, $body, $contact['id'], $url, Protocol::DIASPORA);
}
/** /**
* Receives status messages * Receives status messages
* *
* @param array $importer Array of the importer user * @param array $importer Array of the importer user
* @param SimpleXMLElement $data The message object * @param SimpleXMLElement $data The message object
* @param string $xml The original XML of the message * @param string $xml The original XML of the message
* * @param bool $fetched The message had been fetched and not pushed
* @return int The message id of the newly created item * @return int The message id of the newly created item
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException * @throws \ImagickException
*/ */
private static function receiveStatusMessage(array $importer, SimpleXMLElement $data, $xml) private static function receiveStatusMessage(array $importer, SimpleXMLElement $data, $xml, bool $fetched)
{ {
$author = Strings::escapeTags(XML::unescape($data->author)); $author = Strings::escapeTags(XML::unescape($data->author));
$guid = Strings::escapeTags(XML::unescape($data->guid)); $guid = Strings::escapeTags(XML::unescape($data->guid));
@ -2865,7 +2897,9 @@ class Diaspora
$datarray["protocol"] = Conversation::PARCEL_DIASPORA; $datarray["protocol"] = Conversation::PARCEL_DIASPORA;
$datarray["source"] = $xml; $datarray["source"] = $xml;
if ($datarray["uid"] == 0) { if ($fetched) {
$datarray["post-type"] = Item::PT_FETCHED;
} elseif ($datarray["uid"] == 0) {
$datarray["post-type"] = Item::PT_GLOBAL; $datarray["post-type"] = Item::PT_GLOBAL;
} }
@ -2874,6 +2908,11 @@ class Diaspora
self::storeMentions($datarray['uri-id'], $text); self::storeMentions($datarray['uri-id'], $text);
Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]); Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]);
if (!$fetched && !self::isSolicitedMessage($datarray["uri"], $datarray['uri-id'], $author, $body)) {
DBA::delete('item-uri', ['uri' => $datarray['uri']]);
return false;
}
if ($provider_display_name != "") { if ($provider_display_name != "") {
$datarray["app"] = $provider_display_name; $datarray["app"] = $provider_display_name;
} }

127
src/Protocol/Relay.php Normal file
View file

@ -0,0 +1,127 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Protocol;
use Friendica\Content\Text\BBCode;
use Friendica\Core\Logger;
use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\Search;
/**
* Base class for relay handling
*/
class Relay
{
/**
* Check if a post is wanted
*
* @param array $tags
* @param string $body
* @param int $authorid
* @param string $url
* @return boolean "true" is the post is wanted by the system
*/
public static function isSolicitedPost(array $tags, string $body, int $authorid, string $url, string $network = '')
{
$config = DI::config();
$subscribe = $config->get('system', 'relay_subscribe', false);
if ($subscribe) {
$scope = $config->get('system', 'relay_scope', SR_SCOPE_ALL);
} else {
$scope = SR_SCOPE_NONE;
}
if ($scope == SR_SCOPE_NONE) {
Logger::info('Server does not accept relay posts - rejected', ['network' => $network, 'url' => $url]);
return false;
}
if (Contact::isBlocked($authorid)) {
Logger::info('Author is blocked - rejected', ['author' => $authorid, 'network' => $network, 'url' => $url]);
return false;
}
if (Contact::isHidden($authorid)) {
Logger::info('Author is hidden - rejected', ['author' => $authorid, 'network' => $network, 'url' => $url]);
return false;
}
$systemTags = [];
$userTags = [];
$denyTags = [];
if ($scope == SR_SCOPE_TAGS) {
$server_tags = $config->get('system', 'relay_server_tags');
$tagitems = explode(',', mb_strtolower($server_tags));
foreach ($tagitems AS $tag) {
$systemTags[] = trim($tag, '# ');
}
if ($config->get('system', 'relay_user_tags')) {
$userTags = Search::getUserTags();
}
}
$tagList = array_unique(array_merge($systemTags, $userTags));
$deny_tags = $config->get('system', 'relay_deny_tags');
$tagitems = explode(',', mb_strtolower($deny_tags));
foreach ($tagitems AS $tag) {
$tag = trim($tag, '# ');
$denyTags[] = $tag;
}
if (!empty($tagList) || !empty($denyTags)) {
$content = mb_strtolower(BBCode::toPlaintext($body, false));
foreach ($tags as $tag) {
$tag = mb_strtolower($tag);
if (in_array($tag, $denyTags)) {
Logger::info('Unwanted hashtag found - rejected', ['hashtag' => $tag, 'network' => $network, 'url' => $url]);
return false;
}
if (in_array($tag, $tagList)) {
Logger::info('Subscribed hashtag found - accepted', ['hashtag' => $tag, 'network' => $network, 'url' => $url]);
return true;
}
// We check with "strpos" for performance issues. Only when this is true, the regular expression check is used
// RegExp is taken from here: https://medium.com/@shiba1014/regex-word-boundaries-with-unicode-207794f6e7ed
if ((strpos($content, $tag) !== false) && preg_match('/(?<=[\s,.:;"\']|^)' . preg_quote($tag, '/') . '(?=[\s,.:;"\']|$)/', $content)) {
Logger::info('Subscribed hashtag found in content - accepted', ['hashtag' => $tag, 'network' => $network, 'url' => $url]);
return true;
}
}
}
if ($scope == SR_SCOPE_ALL) {
Logger::info('Server accept all posts - accepted', ['network' => $network, 'url' => $url]);
return true;
}
Logger::info('No matching hashtags found - rejected', ['network' => $network, 'url' => $url]);
return false;
}
}

8
static/settings.config.php
View file

@ -156,10 +156,18 @@ return [
// Periodically (once an hour) run an "optimize table" command for cache tables // Periodically (once an hour) run an "optimize table" command for cache tables
'optimize_tables' => false, 'optimize_tables' => false,
// relay_deny_tags (String)
// Comma separated list of tags that are rejected.
'relay_deny_tags' => '',
// relay_server (String) // relay_server (String)
// Address of the relay server where public posts should be send to. // Address of the relay server where public posts should be send to.
'relay_server' => 'https://social-relay.isurf.ca', 'relay_server' => 'https://social-relay.isurf.ca',
// relay_server_tags (String)
// Comma separated list of tags for the "tags" subscription.
'relay_server_tags' => '',
// relay_user_tags (Boolean) // relay_user_tags (Boolean)
// If enabled, the tags from the saved searches will used for the "tags" subscription in addition to the "relay_server_tags". // If enabled, the tags from the saved searches will used for the "tags" subscription in addition to the "relay_server_tags".
'relay_user_tags' => true, 'relay_user_tags' => true,