Merge pull request #12741 from annando/local-files

Avoid more local links and bad http requests

src/Model/APContact.php

@@ -29,7 +29,6 @@ use Friendica\Core\System;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Item;
-use Friendica\Network\HTTPClient\Client\HttpClientAccept;
 use Friendica\Network\HTTPException;
 use Friendica\Network\Probe;
 use Friendica\Protocol\ActivityNamespace;
@@ -358,20 +357,13 @@ class APContact
 
 		$apcontact['discoverable'] = JsonLD::fetchElement($compacted, 'toot:discoverable', '@value');
 
-		// To-Do
+		if (!empty($apcontact['photo'])) {
-
+			$apcontact['photo'] = trim($apcontact['photo']);
-		// Unhandled
+		}
-		// tag, attachment, image, nomadicLocations, signature, movedTo, liked
-
-		// Unhandled from Misskey
-		// sharedInbox, isCat
-
-		// Unhandled from Kroeg
-		// kroeg:blocks, updated
 
 		if (!empty($apcontact['photo']) && !Network::isValidHttpUrl($apcontact['photo'])) {
-			Logger::info('Invalid URL for photo', ['url' => $apcontact['url'], 'photo' => $apcontact['photo']]);
+			Logger::warning('Invalid URL for photo', ['url' => $apcontact['url'], 'photo' => $apcontact['photo']]);
-			$apcontact['photo'] = null;
+			$apcontact['photo'] = '';
 		}
 
 		// When the photo is too large, try to shorten it by removing parts

src/Model/Contact.php

@@ -2204,13 +2204,18 @@ class Contact
 			return;
 		}
 
+		if (!Network::isValidHttpUrl($avatar)) {
+			Logger::warning('Invalid avatar', ['cid' => $cid, 'avatar' => $avatar]);
+			$avatar = '';
+		}
+
 		$uid = $contact['uid'];
 
 		// Only update the cached photo links of public contacts when they already are cached
 		if (($uid == 0) && !$force && empty($contact['thumb']) && empty($contact['micro']) && !$create_cache) {
 			if (($contact['avatar'] != $avatar) || empty($contact['blurhash'])) {
 				$update_fields = ['avatar' => $avatar];
-				if (!Network::isLocalLink($avatar) && Network::isValidHttpUrl($avatar)) {
+				if (!Network::isLocalLink($avatar)) {
 					$fetchResult = HTTPSignature::fetchRaw($avatar, 0, [HttpClientOptions::ACCEPT_CONTENT => [HttpClientAccept::IMAGE]]);
 
 					$img_str = $fetchResult->getBody();

src/Model/Item.php

@@ -3682,7 +3682,7 @@ class Item
 			return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0;
 		}
 
-		$fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri);
+		$fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', ActivityPub\Receiver::COMPLETION_MANUAL, $uid);
 
 		if ($fetched_uri) {
 			$item_id = self::searchByLink($fetched_uri, $uid);

src/Model/Photo.php

@@ -36,6 +36,7 @@ use Friendica\Object\Image;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Images;
 use Friendica\Security\Security;
+use Friendica\Util\Network;
 use Friendica\Util\Proxy;
 use Friendica\Util\Strings;
 
@@ -582,8 +583,13 @@ class Photo
 
 		$photo_failure = false;
 
+		if (!Network::isValidHttpUrl($image_url)) {
+			Logger::warning('Invalid image url', ['image_url' => $image_url, 'uid' => $uid, 'cid' => $cid, 'callstack' => System::callstack(20)]);
+			return false;
+		}
+
 		$filename = basename($image_url);
-		if (!empty($image_url) && @parse_url($image_url, PHP_URL_HOST)) {
+		if (!empty($image_url)) {
 			$ret = DI::httpClient()->get($image_url, HttpClientAccept::IMAGE);
 			Logger::debug('Got picture', ['Content-Type' => $ret->getHeader('Content-Type'), 'url' => $image_url]);
 			$img_str = $ret->getBody();

src/Model/Tag.php

@@ -194,7 +194,7 @@ class Tag
 			} elseif (Contact::getIdForURL($url, 0, $fetch ? null : false)) {
 				$target = self::ACCOUNT;
 				Logger::debug('URL is an account', ['url' => $url]);
-			} elseif ($fetch && ($target != self::GENERAL_COLLECTION) && Network::isValidHttpUrl($url)) {
+			} elseif ($fetch && ($target != self::GENERAL_COLLECTION)) {
 				$content = ActivityPub::fetchContent($url);
 				if (!empty($content['type']) && ($content['type'] == 'OrderedCollection')) {
 					$target = self::GENERAL_COLLECTION;

src/Network/Probe.php

@@ -121,7 +121,7 @@ class Probe
 		$numeric_fields = ['gsid', 'hide', 'account-type', 'manually-approve'];
 
 		if (!empty($data['photo']) && !Network::isValidHttpUrl($data['photo'])) {
-			Logger::info('Invalid URL for photo', ['url' => $data['url'], 'photo' => $data['photo']]);
+			Logger::warning('Invalid URL for photo', ['url' => $data['url'], 'photo' => $data['photo']]);
 			unset($data['photo']);
 		}
 

src/Protocol/ActivityPub/Processor.php

@@ -1467,13 +1467,14 @@ class Processor
 	 * @param array      $child       activity array with the child of this message
 	 * @param string     $relay_actor Relay actor
 	 * @param int        $completion  Completion mode, see Receiver::COMPLETION_*
+	 * @param int        $uid         User id that is used to fetch the activity
 	 * @return string fetched message URL
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	public static function fetchMissingActivity(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL): string
+	public static function fetchMissingActivity(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL, int $uid = 0): string
 	{
-		$object = self::fetchCachedActivity($url, 0);
+		$object = self::fetchCachedActivity($url, $uid);
 		if (empty($object)) {
 			return '';
 		}
@@ -1536,17 +1537,21 @@ class Processor
 
 		Contact::updateByUrlIfNeeded($actor);
 
-		if (!empty($relay_actor)) {
+		if (!empty($child['thread-completion'])) {
-			$ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor);
-			$ldactivity['completion-mode']   = Receiver::COMPLETION_RELAY;
-		} elseif (!empty($child['thread-completion'])) {
 			$ldactivity['thread-completion'] = $child['thread-completion'];
 			$ldactivity['completion-mode']   = $child['completion-mode'] ?? Receiver::COMPLETION_NONE;
 		} else {
-			$ldactivity['thread-completion'] = Contact::getIdForURL($actor);
+			$ldactivity['thread-completion'] = Contact::getIdForURL($relay_actor ?: $actor);
 			$ldactivity['completion-mode']   = $completion;
 		}
 
+		if ($completion == Receiver::COMPLETION_RELAY) {
+			$ldactivity['from-relay'] = $ldactivity['thread-completion'];
+			if (!self::acceptIncomingMessage($ldactivity, $object['id'])) {
+				return '';
+			}
+		}
+
 		if (!empty($child['thread-children-type'])) {
 			$ldactivity['thread-children-type'] = $child['thread-children-type'];
 		} elseif (!empty($child['type'])) {
@@ -1555,13 +1560,9 @@ class Processor
 			$ldactivity['thread-children-type'] = 'as:Create';
 		}
 
-		if (!empty($relay_actor) && !self::acceptIncomingMessage($ldactivity, $object['id'])) {
-			return '';
-		}
-
 		if (($completion == Receiver::COMPLETION_RELAY) && Queue::exists($url, 'as:Create')) {
 			Logger::notice('Activity has already been queued.', ['url' => $url, 'object' => $activity['id']]);
-		} elseif (ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), 0, true, false, $signer, '', $completion)) {
+		} elseif (ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), $uid, true, false, $signer, '', $completion)) {
 			Logger::notice('Activity had been fetched and processed.', ['url' => $url, 'entry' => $child['entry-id'] ?? 0, 'completion' => $completion, 'object' => $activity['id']]);
 		} else {
 			Logger::notice('Activity had been fetched and will be processed later.', ['url' => $url, 'entry' => $child['entry-id'] ?? 0, 'completion' => $completion, 'object' => $activity['id']]);

src/Protocol/ActivityPub/Queue.php

@@ -236,7 +236,7 @@ class Queue
 		}
 		DBA::close($receivers);
 
-		if (!Receiver::routeActivities($activity, $type, $push, $fetch_parents)) {
+		if (!Receiver::routeActivities($activity, $type, $push, $fetch_parents, $activity['receiver'][0] ?? 0)) {
 			self::remove($activity);
 		}
 

src/Protocol/ActivityPub/Receiver.php

@@ -415,7 +415,6 @@ class Receiver
 			$object_data['object_id'] = JsonLD::fetchElement($activity, 'as:object', '@id');
 			$object_data['object_ids'] = JsonLD::fetchElementArray($activity, 'as:object', '@id');
 			$object_data['content'] = JsonLD::fetchElement($activity, 'as:content', '@type');
-			$object_data['push'] = $push;
 		} elseif (in_array($object_type, self::ACCOUNT_TYPES)) {
 			$object_data = [];
 			$object_data['id'] = JsonLD::fetchElement($activity, '@id');
@@ -423,16 +422,14 @@ class Receiver
 			$object_data['object_actor'] = JsonLD::fetchElement($activity['as:object'], 'as:actor', '@id');
 			$object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object');
 			$object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type');
-			$object_data['push'] = $push;
 			if (!$trust_source && ($type == 'as:Delete')) {
 				$apcontact = APContact::getByURL($object_data['object_id'], true);
 				$trust_source = empty($apcontact) || ($apcontact['type'] == 'Tombstone') || $apcontact['suspended'];
 			}
-		} elseif (in_array($type, ['as:Create', 'as:Update', 'as:Announce', 'as:Invite']) || strpos($type, '#emojiReaction')) {
+		} elseif (in_array($type, ['as:Create', 'as:Update', 'as:Invite']) || strpos($type, '#emojiReaction')) {
 			// Fetch the content only on activities where this matters
 			// We can receive "#emojiReaction" when fetching content from Hubzilla systems
-			// Always fetch on "Announce"
+			$object_data = self::fetchObject($object_id, $activity['as:object'], $trust_source, $fetch_uid);
-			$object_data = self::fetchObject($object_id, $activity['as:object'], $trust_source && ($type != 'as:Announce'), $fetch_uid);
 			if (empty($object_data)) {
 				Logger::info("Object data couldn't be processed");
 				return [];
@@ -440,19 +437,13 @@ class Receiver
 
 			$object_data['object_id'] = $object_id;
 
-			if ($type == 'as:Announce') {
-				$object_data['push'] = false;
-			} else {
-				$object_data['push'] = $push;
-			}
-
 			// Test if it is an answer to a mail
 			if (DBA::exists('mail', ['uri' => $object_data['reply-to-id']])) {
 				$object_data['directmessage'] = true;
 			} else {
 				$object_data['directmessage'] = JsonLD::fetchElement($activity, 'litepub:directMessage');
 			}
-		} elseif (in_array($type, array_merge(self::ACTIVITY_TYPES, ['as:Follow'])) && in_array($object_type, self::CONTENT_TYPES)) {
+		} elseif (in_array($type, array_merge(self::ACTIVITY_TYPES, ['as:Announce', 'as:Follow'])) && in_array($object_type, self::CONTENT_TYPES)) {
 			// Create a mostly empty array out of the activity data (instead of the object).
 			// This way we later don't have to check for the existence of each individual array element.
 			$object_data = self::processObject($activity);
@@ -460,7 +451,6 @@ class Receiver
 			$object_data['author'] = JsonLD::fetchElement($activity, 'as:actor', '@id');
 			$object_data['object_id'] = $object_id;
 			$object_data['object_type'] = ''; // Since we don't fetch the object, we don't know the type
-			$object_data['push'] = $push;
 		} elseif (in_array($type, ['as:Add', 'as:Remove', 'as:Move'])) {
 			$object_data = [];
 			$object_data['id'] = JsonLD::fetchElement($activity, '@id');
@@ -468,7 +458,6 @@ class Receiver
 			$object_data['object_id'] = JsonLD::fetchElement($activity, 'as:object', '@id');
 			$object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type');
 			$object_data['object_content'] = JsonLD::fetchElement($activity['as:object'], 'as:content', '@type');
-			$object_data['push'] = $push;
 		} else {
 			$object_data = [];
 			$object_data['id'] = JsonLD::fetchElement($activity, '@id');
@@ -476,7 +465,6 @@ class Receiver
 			$object_data['object_actor'] = JsonLD::fetchElement($activity['as:object'], 'as:actor', '@id');
 			$object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object');
 			$object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type');
-			$object_data['push'] = $push;
 
 			// An Undo is done on the object of an object, so we need that type as well
 			if (($type == 'as:Undo') && !empty($object_data['object_object'])) {
@@ -491,6 +479,8 @@ class Receiver
 			}
 		}
 
+		$object_data['push'] = $push;
+
 		$object_data = self::addActivityFields($object_data, $activity);
 
 		if (empty($object_data['object_type'])) {
@@ -653,7 +643,9 @@ class Receiver
 			}
 		}
 
-		if (DI::config()->get('system', 'decoupled_receiver') && ($trust_source || DI::config()->get('debug', 'ap_inbox_store_untrusted'))) {
+		$decouple = DI::config()->get('system', 'decoupled_receiver') && !in_array($completion, [self::COMPLETION_MANUAL, self::COMPLETION_ANNOUCE]);
+
+		if ($decouple && ($trust_source || DI::config()->get('debug', 'ap_inbox_store_untrusted'))) {
 			$object_data = Queue::add($object_data, $type, $uid, $http_signer, $push, $trust_source);
 		}
 
@@ -662,7 +654,7 @@ class Receiver
 			return true;
 		}
 
-		if (!empty($object_data['entry-id']) && DI::config()->get('system', 'decoupled_receiver') && ($push || ($completion == self::COMPLETION_RELAY))) {
+		if (!empty($object_data['entry-id']) && $decouple && ($push || ($completion == self::COMPLETION_RELAY))) {
 			if (Queue::isProcessable($object_data['entry-id'])) {
 				// We delay by 5 seconds to allow to accumulate all receivers
 				$delayed = date(DateTimeFormat::MYSQL, time() + 5);
@@ -679,7 +671,7 @@ class Receiver
 			$object_data['recursion-depth'] = $activity['recursion-depth'];
 		}
 
-		if (!self::routeActivities($object_data, $type, $push)) {
+		if (!self::routeActivities($object_data, $type, $push, true, $uid)) {
 			self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer);
 			Queue::remove($object_data);
 		}
@@ -693,10 +685,11 @@ class Receiver
 	 * @param string $type
 	 * @param bool   $push
 	 * @param bool   $fetch_parents
+	 * @param int    $uid
 	 *
 	 * @return boolean Could the activity be routed?
 	 */
-	public static function routeActivities(array $object_data, string $type, bool $push, bool $fetch_parents = true): bool
+	public static function routeActivities(array $object_data, string $type, bool $push, bool $fetch_parents = true, int $uid = 0): bool
 	{
 		$activity = $object_data['object_activity']	?? [];
 
@@ -739,43 +732,24 @@ class Receiver
 
 			case 'as:Announce':
 				if (in_array($object_data['object_type'], self::CONTENT_TYPES)) {
-					$actor = JsonLD::fetchElement($activity, 'as:actor', '@id');
+					if (!Item::searchByLink($object_data['object_id'], $uid)) {
-					$object_data['thread-completion'] = Contact::getIdForURL($actor);
+						if (ActivityPub\Processor::fetchMissingActivity($object_data['object_id'], [], $object_data['actor'], self::COMPLETION_ANNOUCE, $uid)) {
-					$object_data['completion-mode']   = self::COMPLETION_ANNOUCE;
+							Logger::debug('Created announced id', ['uid' => $uid, 'id' => $object_data['object_id']]);
-
+							Queue::remove($object_data);
-					if (!Post::exists(['uri' => $object_data['id'], 'uid' => 0])) {
+						} else {
-						$item = ActivityPub\Processor::createItem($object_data, $fetch_parents);
+							Logger::debug('Announced id was not created', ['uid' => $uid, 'id' => $object_data['object_id']]);
-						if (empty($item)) {
-							Logger::debug('announced id was not created', ['id' => $object_data['id']]);
 							Queue::remove($object_data);
 							return true;
 						}
-
-						$item['post-reason'] = Item::PR_ANNOUNCEMENT;
-						ActivityPub\Processor::postItem($object_data, $item);
-						Logger::debug('Created announced id', ['id' => $object_data['id']]);
 					} else {
-						Logger::info('Announced id already exists', ['id' => $object_data['id']]);
+						Logger::info('Announced id already exists', ['uid' => $uid, 'id' => $object_data['object_id']]);
 						Queue::remove($object_data);
 					}
 
-					if (!empty($activity)) {
+					ActivityPub\Processor::createActivity($object_data, Activity::ANNOUNCE);
-						$announce_object_data = self::processObject($activity);
+				} elseif (in_array($object_data['object_type'], ['as:Tombstone', ''])) {
-						$announce_object_data['name'] = $type;
+					// We don't have the object here or it is deleted. We ignore this activity.
-						$announce_object_data['author'] = $actor;
+					Queue::remove($object_data);
-						$announce_object_data['object_id'] = $object_data['id'];
-						$announce_object_data['object_type'] = $object_data['object_type'];
-						$announce_object_data['push'] = $push;
-						Logger::debug('Create announce activity', ['id' => $announce_object_data['id'], 'object_data' => $announce_object_data]);
-
-						if (!empty($object_data['raw'])) {
-							$announce_object_data['raw'] = $object_data['raw'];
-						}
-						if (!empty($object_data['raw-object'])) {
-							$announce_object_data['raw-object'] = $object_data['raw-object'];
-						}
-						ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE);
-					}
 				} else {
 					return false;
 				}
@@ -1066,7 +1040,7 @@ class Receiver
 
 			foreach ($receiver_list as $receiver) {
 				if ($receiver == 'Public') {
-					Logger::notice('Not compacted public collection found', ['activity' => $activity, 'callstack' => System::callstack(20)]);
+					Logger::warning('Not compacted public collection found', ['activity' => $activity, 'callstack' => System::callstack(20)]);
 					$receiver = ActivityPub::PUBLIC_COLLECTION;
 				}
 				if ($receiver == self::PUBLIC_COLLECTION) {
@@ -1474,14 +1448,6 @@ class Receiver
 			return $object_data;
 		}
 
-		if ($type == 'as:Announce') {
-			$object_id = JsonLD::fetchElement($object, 'object', '@id');
-			if (empty($object_id) || !is_string($object_id)) {
-				return false;
-			}
-			return self::fetchObject($object_id, [], false, $uid);
-		}
-
 		Logger::info('Unhandled object type: ' . $type);
 		return false;
 	}
@@ -1899,7 +1865,7 @@ class Receiver
 			$object_data['reply-to-id'] = $object_data['id'];
 
 			// On activities the "reply to" is the id of the object it refers to
-			if (in_array($object_data['object_type'], self::ACTIVITY_TYPES)) {
+			if (in_array($object_data['object_type'], array_merge(self::ACTIVITY_TYPES, ['as:Announce']))) {
 				$object_id = JsonLD::fetchElement($object, 'as:object', '@id');
 				if (!empty($object_id)) {
 					$object_data['reply-to-id'] = $object_id;

src/Worker/OnePoll.php

@@ -159,7 +159,7 @@ class OnePoll
 		}
 
 		if (!Network::isValidHttpUrl($contact['poll'])) {
-			Logger::notice('Poll address is not valid', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'poll' => $contact['poll']]);
+			Logger::warning('Poll address is not valid', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'poll' => $contact['poll']]);
 			return false;
 		}
 

src/Worker/PollContacts.php

@@ -41,7 +41,7 @@ class PollContacts
 			$abandon_days = 0;
 		}
 
-		$condition = ['network' => [Protocol::FEED, Protocol::MAIL, Protocol::OSTATUS], 'self' => false, 'blocked' => false];
+		$condition = ['network' => [Protocol::FEED, Protocol::MAIL, Protocol::OSTATUS], 'self' => false, 'blocked' => false, 'archive' => false];
 
 		if (!empty($abandon_days)) {
 			$condition = DBA::mergeConditions($condition,

tests/FixtureTest.php

@@ -82,7 +82,10 @@ abstract class FixtureTest extends DatabaseTest
 
 		$dba->setTestmode(true);
 
+		if (DI::lock()->acquire('Test-checkInitialValues', 0)) {
 			DBStructure::checkInitialValues();
+			DI::lock()->release('Test-checkInitialValues');
+		}
 
 		// Load the API dataset for the whole API
 		$this->loadFixture(__DIR__ . '/datasets/api.fixture.php', $dba);