From e26d73393e0c65d2c9544763210739dbe6614582 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 30 Sep 2020 17:37:46 +0000 Subject: [PATCH] New "relay" class / check of incoming popsts from DFRN and Diaspora --- src/Protocol/ActivityPub/Processor.php | 73 +--------------- src/Protocol/DFRN.php | 27 ++++++ src/Protocol/Diaspora.php | 63 +++++++++++--- src/Protocol/Relay.php | 114 +++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 82 deletions(-) create mode 100644 src/Protocol/Relay.php diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 75d47090b..ab1b27d33 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -35,15 +35,14 @@ use Friendica\Model\Event; use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\Mail; -use Friendica\Model\Search; use Friendica\Model\Tag; use Friendica\Model\User; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; +use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; use Friendica\Util\JsonLD; use Friendica\Util\Strings; -use Text_LanguageDetect; /** * ActivityPub Processor Protocol class @@ -801,25 +800,13 @@ class Processor return true; } - $config = DI::config(); - - $subscribe = $config->get('system', 'relay_subscribe', false); - if ($subscribe) { - $scope = $config->get('system', 'relay_scope', SR_SCOPE_ALL); - } else { - $scope = SR_SCOPE_NONE; - } - $replyto = JsonLD::fetchElement($activity['as:object'], 'as:inReplyTo', '@id'); if (Item::exists(['uri' => $replyto])) { Logger::info('Post is a reply to an existing post - accepted', ['id' => $id, 'replyto' => $replyto]); return true; } - if ($scope == SR_SCOPE_NONE) { - Logger::info('Server does not accept relay posts - rejected', ['id' => $id]); - return false; - } + $body = HTML::toBBCode(JsonLD::fetchElement($activity['as:object'], 'as:content', '@value')); $messageTags = []; $tags = Receiver::processTags(JsonLD::fetchElementArray($activity['as:object'], 'as:tag') ?? []); @@ -832,61 +819,7 @@ class Processor } } - $systemTags = []; - $userTags = []; - $denyTags = []; - - if ($scope == SR_SCOPE_TAGS) { - $server_tags = $config->get('system', 'relay_server_tags'); - $tagitems = explode(',', mb_strtolower($server_tags)); - foreach ($tagitems AS $tag) { - $systemTags[] = trim($tag, '# '); - } - - if ($config->get('system', 'relay_user_tags')) { - $userTags = Search::getUserTags(); - } - } - - $tagList = array_unique(array_merge($systemTags, $userTags)); - - $deny_tags = $config->get('system', 'relay_deny_tags'); - $tagitems = explode(',', mb_strtolower($deny_tags)); - foreach ($tagitems AS $tag) { - $tag = trim($tag, '# '); - $denyTags[] = $tag; - } - - if (!empty($tagList) || !empty($denyTags)) { - $content = mb_strtolower(BBCode::toPlaintext(HTML::toBBCode(JsonLD::fetchElement($activity['as:object'], 'as:content', '@value')), false)); - - foreach ($messageTags as $tag) { - if (in_array($tag, $denyTags)) { - Logger::info('Unwanted hashtag found - rejected', ['id' => $id, 'hashtag' => $tag]); - return false; - } - - if (in_array($tag, $tagList)) { - Logger::info('Subscribed hashtag found - accepted', ['id' => $id, 'hashtag' => $tag]); - return true; - } - - // We check with "strpos" for performance issues. Only when this is true, the regular expression check is used - // RegExp is taken from here: https://medium.com/@shiba1014/regex-word-boundaries-with-unicode-207794f6e7ed - if ((strpos($content, $tag) !== false) && preg_match('/(?<=[\s,.:;"\']|^)' . preg_quote($tag, '/') . '(?=[\s,.:;"\']|$)/', $content)) { - Logger::info('Subscribed hashtag found in content - accepted', ['id' => $id, 'hashtag' => $tag]); - return true; - } - } - } - - if ($scope == SR_SCOPE_ALL) { - Logger::info('Server accept all posts - accepted', ['id' => $id]); - return true; - } - - Logger::info('No matching hashtags found - rejected', ['id' => $id]); - return false; + return Relay::isSolicitedPost($messageTags, $body, $id, Protocol::ACTIVITYPUB); } /** diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 78843a3f1..e4e71fd03 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2261,6 +2261,25 @@ class DFRN } } + /** + * Checks if an incoming message is wanted + * + * @param array $item + * @return boolean Is the message wanted? + */ + private static function isSolicitedMessage(array $item) + { + if (DBA::exists('contact', ["`nurl` = ? AND `uid` != ? AND `rel` IN (?, ?)", + Strings::normaliseLink($item["author-link"]), 0, Contact::FRIEND, Contact::SHARING])) { + Logger::info('Author has got followers - accepted', ['uri' => $item['uri'], 'author' => $item["author-link"]]); + return true; + } + + $taglist = Tag::getByURIId($item['uri-id'], [Tag::HASHTAG]); + $tags = array_column($taglist, 'name'); + return Relay::isSolicitedPost($tags, $item['body'], $item['uri'], Protocol::DFRN); + } + /** * Processes the entry elements which contain the items and comments * @@ -2450,6 +2469,14 @@ class DFRN } } + // Check if the message is wanted + if (($importer["importer_uid"] == 0) && ($item['uri'] == $item['parent-uri'])) { + if (!self::isSolicitedMessage($item)) { + DBA::delete('uri-id', ['uri' => $item['uri']]); + return 403; + } + } + // Get the type of the item (Top level post, reply or remote reply) $entrytype = self::getEntryType($importer, $item); diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 6e6ea03fb..592f76ce7 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -639,13 +639,14 @@ class Diaspora /** * Dispatches public messages and find the fitting receivers * - * @param array $msg The post that will be dispatched + * @param array $msg The post that will be dispatched + * @param bool $fetched The message had been fetched (default "false") * * @return int The message id of the generated message, "true" or "false" if there was an error * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function dispatchPublic($msg) + public static function dispatchPublic($msg, bool $fetched = false) { $enabled = intval(DI::config()->get("system", "diaspora_enabled")); if (!$enabled) { @@ -659,7 +660,7 @@ class Diaspora } $importer = ["uid" => 0, "page-flags" => User::PAGE_FLAGS_FREELOVE]; - $success = self::dispatch($importer, $msg, $fields); + $success = self::dispatch($importer, $msg, $fields, $fetched); return $success; } @@ -670,12 +671,13 @@ class Diaspora * @param array $importer Array of the importer user * @param array $msg The post that will be dispatched * @param SimpleXMLElement $fields SimpleXML object that contains the message + * @param bool $fetched The message had been fetched (default "false") * * @return int The message id of the generated message, "true" or "false" if there was an error * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function dispatch(array $importer, $msg, SimpleXMLElement $fields = null) + public static function dispatch(array $importer, $msg, SimpleXMLElement $fields = null, bool $fetched = false) { // The sender is the handle of the contact that sent the message. // This will often be different with relayed messages (for example "like" and "comment") @@ -708,7 +710,7 @@ class Diaspora return self::receiveAccountDeletion($fields); case "comment": - return self::receiveComment($importer, $sender, $fields, $msg["message"]); + return self::receiveComment($importer, $sender, $fields, $msg["message"], $fetched); case "contact": if (!$private) { @@ -761,7 +763,7 @@ class Diaspora return self::receiveRetraction($importer, $sender, $fields); case "status_message": - return self::receiveStatusMessage($importer, $fields, $msg["message"]); + return self::receiveStatusMessage($importer, $fields, $msg["message"], $fetched); default: Logger::log("Unknown message type ".$type); @@ -1238,7 +1240,7 @@ class Diaspora Logger::log("Successfully fetched item ".$guid." from ".$server, Logger::DEBUG); // Now call the dispatcher - return self::dispatchPublic($msg); + return self::dispatchPublic($msg, true); } /** @@ -1674,12 +1676,13 @@ class Diaspora * @param string $sender The sender of the message * @param object $data The message object * @param string $xml The original XML of the message + * @param bool $fetched The message had been fetched and not pushed * * @return int The message id of the generated comment or "false" if there was an error * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - private static function receiveComment(array $importer, $sender, $data, $xml) + private static function receiveComment(array $importer, $sender, $data, $xml, bool $fetched) { $author = Strings::escapeTags(XML::unescape($data->author)); $guid = Strings::escapeTags(XML::unescape($data->guid)); @@ -1736,7 +1739,13 @@ class Diaspora $datarray["owner-id"] = Contact::getIdForURL($contact["url"], 0); // Will be overwritten for sharing accounts in Item::insert - $datarray['post-type'] = ($datarray["uid"] == 0) ? Item::PT_GLOBAL : Item::PT_COMMENT; + if ($fetched) { + $datarray["post-type"] = Item::PT_FETCHED; + } elseif ($datarray["uid"] == 0) { + $datarray["post-type"] = Item::PT_GLOBAL; + } else { + $datarray["post-type"] = Item::PT_COMMENT; + } $datarray["guid"] = $guid; $datarray["uri"] = self::getUriFromGuid($author, $guid); @@ -2778,18 +2787,41 @@ class Diaspora return true; } + /** + * Checks if an incoming message is wanted + * + * @param string $url + * @param integer $uriid + * @param string $author + * @param string $body + * @return boolean Is the message wanted? + */ + private static function isSolicitedMessage(string $url, int $uriid, string $author, string $body) + { + $contact = Contact::getByURL($author); + if (DBA::exists('contact', ["`nurl` = ? AND `uid` != ? AND `rel` IN (?, ?)", + $contact['nurl'], 0, Contact::FRIEND, Contact::SHARING])) { + Logger::info('Author has got followers - accepted', ['url' => $url, 'author' => $author]); + return true; + } + + $taglist = Tag::getByURIId($uriid, [Tag::HASHTAG]); + $tags = array_column($taglist, 'name'); + return Relay::isSolicitedPost($tags, $body, $url, Protocol::DIASPORA); + } + /** * Receives status messages * * @param array $importer Array of the importer user * @param SimpleXMLElement $data The message object * @param string $xml The original XML of the message - * + * @param bool $fetched The message had been fetched and not pushed * @return int The message id of the newly created item * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - private static function receiveStatusMessage(array $importer, SimpleXMLElement $data, $xml) + private static function receiveStatusMessage(array $importer, SimpleXMLElement $data, $xml, bool $fetched) { $author = Strings::escapeTags(XML::unescape($data->author)); $guid = Strings::escapeTags(XML::unescape($data->guid)); @@ -2865,7 +2897,9 @@ class Diaspora $datarray["protocol"] = Conversation::PARCEL_DIASPORA; $datarray["source"] = $xml; - if ($datarray["uid"] == 0) { + if ($fetched) { + $datarray["post-type"] = Item::PT_FETCHED; + } elseif ($datarray["uid"] == 0) { $datarray["post-type"] = Item::PT_GLOBAL; } @@ -2874,6 +2908,11 @@ class Diaspora self::storeMentions($datarray['uri-id'], $text); Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]); + if (!$fetched && !self::isSolicitedMessage($datarray["uri"], $datarray['uri-id'], $author, $body)) { + DBA::delete('uri-id', ['uri' => $datarray['uri']]); + return false; + } + if ($provider_display_name != "") { $datarray["app"] = $provider_display_name; } diff --git a/src/Protocol/Relay.php b/src/Protocol/Relay.php new file mode 100644 index 000000000..210509d26 --- /dev/null +++ b/src/Protocol/Relay.php @@ -0,0 +1,114 @@ +. + * + */ + +namespace Friendica\Protocol; + +use Friendica\Content\Text\BBCode; +use Friendica\Core\Logger; +use Friendica\DI; +use Friendica\Model\Search; + +/** + * Base class for relay handling + */ +final class Relay +{ + /** + * Check if a post is wanted + * + * @param array $tags + * @param string $body + * @param string $url + * @return boolean "true" is the post is wanted by the system + */ + public static function isSolicitedPost(array $tags, string $body, string $url, string $network = '') + { + $config = DI::config(); + + $subscribe = $config->get('system', 'relay_subscribe', false); + if ($subscribe) { + $scope = $config->get('system', 'relay_scope', SR_SCOPE_ALL); + } else { + $scope = SR_SCOPE_NONE; + } + + if ($scope == SR_SCOPE_NONE) { + Logger::info('Server does not accept relay posts - rejected', ['network' => $network, 'url' => $url]); + return false; + } + + $systemTags = []; + $userTags = []; + $denyTags = []; + + if ($scope == SR_SCOPE_TAGS) { + $server_tags = $config->get('system', 'relay_server_tags'); + $tagitems = explode(',', mb_strtolower($server_tags)); + foreach ($tagitems AS $tag) { + $systemTags[] = trim($tag, '# '); + } + + if ($config->get('system', 'relay_user_tags')) { + $userTags = Search::getUserTags(); + } + } + + $tagList = array_unique(array_merge($systemTags, $userTags)); + + $deny_tags = $config->get('system', 'relay_deny_tags'); + $tagitems = explode(',', mb_strtolower($deny_tags)); + foreach ($tagitems AS $tag) { + $tag = trim($tag, '# '); + $denyTags[] = $tag; + } + + if (!empty($tagList) || !empty($denyTags)) { + $content = mb_strtolower(BBCode::toPlaintext($body, false)); + + foreach ($tags as $tag) { + if (in_array($tag, $denyTags)) { + Logger::info('Unwanted hashtag found - rejected', ['hashtag' => $tag, 'network' => $network, 'url' => $url]); + return false; + } + + if (in_array($tag, $tagList)) { + Logger::info('Subscribed hashtag found - accepted', ['hashtag' => $tag, 'network' => $network, 'url' => $url]); + return true; + } + + // We check with "strpos" for performance issues. Only when this is true, the regular expression check is used + // RegExp is taken from here: https://medium.com/@shiba1014/regex-word-boundaries-with-unicode-207794f6e7ed + if ((strpos($content, $tag) !== false) && preg_match('/(?<=[\s,.:;"\']|^)' . preg_quote($tag, '/') . '(?=[\s,.:;"\']|$)/', $content)) { + Logger::info('Subscribed hashtag found in content - accepted', ['hashtag' => $tag, 'network' => $network, 'url' => $url]); + return true; + } + } + } + + if ($scope == SR_SCOPE_ALL) { + Logger::info('Server accept all posts - accepted', ['network' => $network, 'url' => $url]); + return true; + } + + Logger::info('No matching hashtags found - rejected', ['network' => $network, 'url' => $url]); + return false; + } +}