New "relay" class / check of incoming popsts from DFRN and Diaspora

src/Protocol/ActivityPub/Processor.php

@@ -35,15 +35,14 @@ use Friendica\Model\Event;
 use Friendica\Model\Item;
 use Friendica\Model\ItemURI;
 use Friendica\Model\Mail;
-use Friendica\Model\Search;
 use Friendica\Model\Tag;
 use Friendica\Model\User;
 use Friendica\Protocol\Activity;
 use Friendica\Protocol\ActivityPub;
+use Friendica\Protocol\Relay;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\JsonLD;
 use Friendica\Util\Strings;
-use Text_LanguageDetect;
 
 /**
  * ActivityPub Processor Protocol class
@@ -801,25 +800,13 @@ class Processor
 			return true;
 		}
 
-		$config = DI::config();
-
-		$subscribe = $config->get('system', 'relay_subscribe', false);
-		if ($subscribe) {
-			$scope = $config->get('system', 'relay_scope', SR_SCOPE_ALL);
-		} else {
-			$scope = SR_SCOPE_NONE;
-		}
-
 		$replyto = JsonLD::fetchElement($activity['as:object'], 'as:inReplyTo', '@id');
 		if (Item::exists(['uri' => $replyto])) {
 			Logger::info('Post is a reply to an existing post - accepted', ['id' => $id, 'replyto' => $replyto]);
 			return true;
 		}
 
-		if ($scope == SR_SCOPE_NONE) {
+		$body = HTML::toBBCode(JsonLD::fetchElement($activity['as:object'], 'as:content', '@value'));
-			Logger::info('Server does not accept relay posts - rejected', ['id' => $id]);
-			return false;
-		}
 
 		$messageTags = [];
 		$tags = Receiver::processTags(JsonLD::fetchElementArray($activity['as:object'], 'as:tag') ?? []);
@@ -832,61 +819,7 @@ class Processor
 			}
 		}
 
-		$systemTags = [];
+		return Relay::isSolicitedPost($messageTags, $body, $id, Protocol::ACTIVITYPUB);
-		$userTags = [];
-		$denyTags = [];
-
-		if ($scope == SR_SCOPE_TAGS) {
-			$server_tags = $config->get('system', 'relay_server_tags');
-			$tagitems = explode(',', mb_strtolower($server_tags));
-			foreach ($tagitems AS $tag) {
-				$systemTags[] = trim($tag, '# ');
-			}
-
-			if ($config->get('system', 'relay_user_tags')) {
-				$userTags = Search::getUserTags();
-			}
-		}
-
-		$tagList = array_unique(array_merge($systemTags, $userTags));
-
-		$deny_tags = $config->get('system', 'relay_deny_tags');
-		$tagitems = explode(',', mb_strtolower($deny_tags));
-		foreach ($tagitems AS $tag) {
-			$tag = trim($tag, '# ');
-			$denyTags[] = $tag;
-		}
-
-		if (!empty($tagList) || !empty($denyTags)) {
-			$content = mb_strtolower(BBCode::toPlaintext(HTML::toBBCode(JsonLD::fetchElement($activity['as:object'], 'as:content', '@value')), false));
-
-			foreach ($messageTags as $tag) {
-				if (in_array($tag, $denyTags)) {
-					Logger::info('Unwanted hashtag found - rejected', ['id' => $id, 'hashtag' => $tag]);
-					return false;
-				}
-
-				if (in_array($tag, $tagList)) {
-					Logger::info('Subscribed hashtag found - accepted', ['id' => $id, 'hashtag' => $tag]);
-					return true;
-				}
-
-				// We check with "strpos" for performance issues. Only when this is true, the regular expression check is used
-				// RegExp is taken from here: https://medium.com/@shiba1014/regex-word-boundaries-with-unicode-207794f6e7ed
-				if ((strpos($content, $tag) !== false) && preg_match('/(?<=[\s,.:;"\']|^)' . preg_quote($tag, '/') . '(?=[\s,.:;"\']|$)/', $content)) {
-					Logger::info('Subscribed hashtag found in content - accepted', ['id' => $id, 'hashtag' => $tag]);
-					return true;
-				}
-			}
-		}
-
-		if ($scope == SR_SCOPE_ALL) {
-			Logger::info('Server accept all posts - accepted', ['id' => $id]);
-			return true;
-		}
-
-		Logger::info('No matching hashtags found - rejected', ['id' => $id]);
-		return false;
 	}
 
 	/**

src/Protocol/DFRN.php

@@ -2261,6 +2261,25 @@ class DFRN
 		}
 	}
 
+	/**
+	 * Checks if an incoming message is wanted
+	 *
+	 * @param array $item
+	 * @return boolean Is the message wanted?
+	 */
+	private static function isSolicitedMessage(array $item)
+	{
+		if (DBA::exists('contact', ["`nurl` = ? AND `uid` != ? AND `rel` IN (?, ?)",
+			Strings::normaliseLink($item["author-link"]), 0, Contact::FRIEND, Contact::SHARING])) {
+			Logger::info('Author has got followers - accepted', ['uri' => $item['uri'], 'author' => $item["author-link"]]);
+			return true;
+		}
+
+		$taglist = Tag::getByURIId($item['uri-id'], [Tag::HASHTAG]);
+		$tags = array_column($taglist, 'name');
+		return Relay::isSolicitedPost($tags, $item['body'], $item['uri'], Protocol::DFRN);
+	}
+
 	/**
 	 * Processes the entry elements which contain the items and comments
 	 *
@@ -2450,6 +2469,14 @@ class DFRN
 			}
 		}
 
+		// Check if the message is wanted
+		if (($importer["importer_uid"] == 0) && ($item['uri'] == $item['parent-uri'])) {
+			if (!self::isSolicitedMessage($item)) {
+				DBA::delete('uri-id', ['uri' => $item['uri']]);
+				return 403;
+			}
+		}
+				
 		// Get the type of the item (Top level post, reply or remote reply)
 		$entrytype = self::getEntryType($importer, $item);
 

src/Protocol/Diaspora.php

@@ -639,13 +639,14 @@ class Diaspora
 	/**
 	 * Dispatches public messages and find the fitting receivers
 	 *
-	 * @param array $msg The post that will be dispatched
+	 * @param array $msg     The post that will be dispatched
+	 * @param bool  $fetched The message had been fetched (default "false")
 	 *
 	 * @return int The message id of the generated message, "true" or "false" if there was an error
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	public static function dispatchPublic($msg)
+	public static function dispatchPublic($msg, bool $fetched = false)
 	{
 		$enabled = intval(DI::config()->get("system", "diaspora_enabled"));
 		if (!$enabled) {
@@ -659,7 +660,7 @@ class Diaspora
 		}
 
 		$importer = ["uid" => 0, "page-flags" => User::PAGE_FLAGS_FREELOVE];
-		$success = self::dispatch($importer, $msg, $fields);
+		$success = self::dispatch($importer, $msg, $fields, $fetched);
 
 		return $success;
 	}
@@ -670,12 +671,13 @@ class Diaspora
 	 * @param array            $importer Array of the importer user
 	 * @param array            $msg      The post that will be dispatched
 	 * @param SimpleXMLElement $fields   SimpleXML object that contains the message
+	 * @param bool             $fetched  The message had been fetched (default "false")
 	 *
 	 * @return int The message id of the generated message, "true" or "false" if there was an error
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	public static function dispatch(array $importer, $msg, SimpleXMLElement $fields = null)
+	public static function dispatch(array $importer, $msg, SimpleXMLElement $fields = null, bool $fetched = false)
 	{
 		// The sender is the handle of the contact that sent the message.
 		// This will often be different with relayed messages (for example "like" and "comment")
@@ -708,7 +710,7 @@ class Diaspora
 				return self::receiveAccountDeletion($fields);
 
 			case "comment":
-				return self::receiveComment($importer, $sender, $fields, $msg["message"]);
+				return self::receiveComment($importer, $sender, $fields, $msg["message"], $fetched);
 
 			case "contact":
 				if (!$private) {
@@ -761,7 +763,7 @@ class Diaspora
 				return self::receiveRetraction($importer, $sender, $fields);
 
 			case "status_message":
-				return self::receiveStatusMessage($importer, $fields, $msg["message"]);
+				return self::receiveStatusMessage($importer, $fields, $msg["message"], $fetched);
 
 			default:
 				Logger::log("Unknown message type ".$type);
@@ -1238,7 +1240,7 @@ class Diaspora
 		Logger::log("Successfully fetched item ".$guid." from ".$server, Logger::DEBUG);
 
 		// Now call the dispatcher
-		return self::dispatchPublic($msg);
+		return self::dispatchPublic($msg, true);
 	}
 
 	/**
@@ -1674,12 +1676,13 @@ class Diaspora
 	 * @param string $sender   The sender of the message
 	 * @param object $data     The message object
 	 * @param string $xml      The original XML of the message
+	 * @param bool   $fetched  The message had been fetched and not pushed
 	 *
 	 * @return int The message id of the generated comment or "false" if there was an error
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	private static function receiveComment(array $importer, $sender, $data, $xml)
+	private static function receiveComment(array $importer, $sender, $data, $xml, bool $fetched)
 	{
 		$author = Strings::escapeTags(XML::unescape($data->author));
 		$guid = Strings::escapeTags(XML::unescape($data->guid));
@@ -1736,7 +1739,13 @@ class Diaspora
 		$datarray["owner-id"] = Contact::getIdForURL($contact["url"], 0);
 
 		// Will be overwritten for sharing accounts in Item::insert
-		$datarray['post-type'] = ($datarray["uid"] == 0) ? Item::PT_GLOBAL : Item::PT_COMMENT;
+		if ($fetched) {
+			$datarray["post-type"] = Item::PT_FETCHED;
+		} elseif ($datarray["uid"] == 0) {
+			$datarray["post-type"] = Item::PT_GLOBAL;
+		} else {
+			$datarray["post-type"] = Item::PT_COMMENT;
+		}
 
 		$datarray["guid"] = $guid;
 		$datarray["uri"] = self::getUriFromGuid($author, $guid);
@@ -2778,18 +2787,41 @@ class Diaspora
 		return true;
 	}
 
+	/**
+	 * Checks if an incoming message is wanted
+	 *
+	 * @param string $url
+	 * @param integer $uriid
+	 * @param string $author
+	 * @param string $body
+	 * @return boolean Is the message wanted?
+	 */
+	private static function isSolicitedMessage(string $url, int $uriid, string $author, string $body)
+	{
+		$contact = Contact::getByURL($author);
+		if (DBA::exists('contact', ["`nurl` = ? AND `uid` != ? AND `rel` IN (?, ?)",
+			$contact['nurl'], 0, Contact::FRIEND, Contact::SHARING])) {
+			Logger::info('Author has got followers - accepted', ['url' => $url, 'author' => $author]);
+			return true;
+		}
+
+		$taglist = Tag::getByURIId($uriid, [Tag::HASHTAG]);
+		$tags = array_column($taglist, 'name');
+		return Relay::isSolicitedPost($tags, $body, $url, Protocol::DIASPORA);
+	}
+
 	/**
 	 * Receives status messages
 	 *
 	 * @param array            $importer Array of the importer user
 	 * @param SimpleXMLElement $data     The message object
 	 * @param string           $xml      The original XML of the message
-	 *
+	 * @param bool             $fetched  The message had been fetched and not pushed
 	 * @return int The message id of the newly created item
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	private static function receiveStatusMessage(array $importer, SimpleXMLElement $data, $xml)
+	private static function receiveStatusMessage(array $importer, SimpleXMLElement $data, $xml, bool $fetched)
 	{
 		$author = Strings::escapeTags(XML::unescape($data->author));
 		$guid = Strings::escapeTags(XML::unescape($data->guid));
@@ -2865,7 +2897,9 @@ class Diaspora
 		$datarray["protocol"] = Conversation::PARCEL_DIASPORA;
 		$datarray["source"] = $xml;
 
-		if ($datarray["uid"] == 0) {
+		if ($fetched) {
+			$datarray["post-type"] = Item::PT_FETCHED;
+		} elseif ($datarray["uid"] == 0) {
 			$datarray["post-type"] = Item::PT_GLOBAL;
 		}
 
@@ -2874,6 +2908,11 @@ class Diaspora
 		self::storeMentions($datarray['uri-id'], $text);
 		Tag::storeRawTagsFromBody($datarray['uri-id'], $datarray["body"]);
 
+		if (!$fetched && !self::isSolicitedMessage($datarray["uri"], $datarray['uri-id'], $author, $body)) {
+			DBA::delete('uri-id', ['uri' => $datarray['uri']]);
+			return false;
+		}
+
 		if ($provider_display_name != "") {
 			$datarray["app"] = $provider_display_name;
 		}

src/Protocol/Relay.php

@@ -0,0 +1,114 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Protocol;
+
+use Friendica\Content\Text\BBCode;
+use Friendica\Core\Logger;
+use Friendica\DI;
+use Friendica\Model\Search;
+
+/**
+ * Base class for relay handling
+ */
+final class Relay
+{
+	/**
+	 * Check if a post is wanted
+	 *
+	 * @param array $tags
+	 * @param string $body
+	 * @param string $url
+	 * @return boolean "true" is the post is wanted by the system
+	 */
+	public static function isSolicitedPost(array $tags, string $body, string $url, string $network = '')
+	{
+		$config = DI::config();
+
+		$subscribe = $config->get('system', 'relay_subscribe', false);
+		if ($subscribe) {
+			$scope = $config->get('system', 'relay_scope', SR_SCOPE_ALL);
+		} else {
+			$scope = SR_SCOPE_NONE;
+		}
+
+		if ($scope == SR_SCOPE_NONE) {
+			Logger::info('Server does not accept relay posts - rejected', ['network' => $network, 'url' => $url]);
+			return false;
+		}
+
+		$systemTags = [];
+		$userTags = [];
+		$denyTags = [];
+
+		if ($scope == SR_SCOPE_TAGS) {
+			$server_tags = $config->get('system', 'relay_server_tags');
+			$tagitems = explode(',', mb_strtolower($server_tags));
+			foreach ($tagitems AS $tag) {
+				$systemTags[] = trim($tag, '# ');
+			}
+
+			if ($config->get('system', 'relay_user_tags')) {
+				$userTags = Search::getUserTags();
+			}
+		}
+
+		$tagList = array_unique(array_merge($systemTags, $userTags));
+
+		$deny_tags = $config->get('system', 'relay_deny_tags');
+		$tagitems = explode(',', mb_strtolower($deny_tags));
+		foreach ($tagitems AS $tag) {
+			$tag = trim($tag, '# ');
+			$denyTags[] = $tag;
+		}
+
+		if (!empty($tagList) || !empty($denyTags)) {
+			$content = mb_strtolower(BBCode::toPlaintext($body, false));
+
+			foreach ($tags as $tag) {
+				if (in_array($tag, $denyTags)) {
+					Logger::info('Unwanted hashtag found - rejected', ['hashtag' => $tag, 'network' => $network, 'url' => $url]);
+					return false;
+				}
+
+				if (in_array($tag, $tagList)) {
+					Logger::info('Subscribed hashtag found - accepted', ['hashtag' => $tag, 'network' => $network, 'url' => $url]);
+					return true;
+				}
+
+				// We check with "strpos" for performance issues. Only when this is true, the regular expression check is used
+				// RegExp is taken from here: https://medium.com/@shiba1014/regex-word-boundaries-with-unicode-207794f6e7ed
+				if ((strpos($content, $tag) !== false) && preg_match('/(?<=[\s,.:;"\']|^)' . preg_quote($tag, '/') . '(?=[\s,.:;"\']|$)/', $content)) {
+					Logger::info('Subscribed hashtag found in content - accepted', ['hashtag' => $tag, 'network' => $network, 'url' => $url]);
+					return true;
+				}
+			}
+		}
+
+		if ($scope == SR_SCOPE_ALL) {
+			Logger::info('Server accept all posts - accepted', ['network' => $network, 'url' => $url]);
+			return true;
+		}
+
+		Logger::info('No matching hashtags found - rejected', ['network' => $network, 'url' => $url]);
+		return false;
+	}
+}