From 999cdc7db90678c37eeb1f5874d993faa8ab748f Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 28 Jan 2023 14:57:04 +0000 Subject: [PATCH] Avoid more local links and bad http requests --- src/Model/APContact.php | 7 ++- src/Model/Contact.php | 7 ++- src/Model/Item.php | 2 +- src/Model/Photo.php | 8 ++- src/Model/Tag.php | 2 +- src/Network/Probe.php | 2 +- src/Protocol/ActivityPub/Processor.php | 25 ++++---- src/Protocol/ActivityPub/Queue.php | 2 +- src/Protocol/ActivityPub/Receiver.php | 84 ++++++++------------------ src/Worker/OnePoll.php | 2 +- src/Worker/PollContacts.php | 2 +- 11 files changed, 62 insertions(+), 81 deletions(-) diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 1fedfc0c79..104629da47 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -368,10 +368,13 @@ class APContact // Unhandled from Kroeg // kroeg:blocks, updated + if (!empty($apcontact['photo'])) { + $apcontact['photo'] = trim($apcontact['photo']); + } if (!empty($apcontact['photo']) && !Network::isValidHttpUrl($apcontact['photo'])) { - Logger::info('Invalid URL for photo', ['url' => $apcontact['url'], 'photo' => $apcontact['photo']]); - $apcontact['photo'] = null; + Logger::warning('Invalid URL for photo', ['url' => $apcontact['url'], 'photo' => $apcontact['photo']]); + $apcontact['photo'] = ''; } // When the photo is too large, try to shorten it by removing parts diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 1408e4a0cb..0ff29f8886 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -2204,13 +2204,18 @@ class Contact return; } + if (!Network::isValidHttpUrl($avatar)) { + Logger::warning('Invalid avatar', ['cid' => $cid, 'avatar' => $avatar]); + $avatar = ''; + } + $uid = $contact['uid']; // Only update the cached photo links of public contacts when they already are cached if (($uid == 0) && !$force && empty($contact['thumb']) && empty($contact['micro']) && !$create_cache) { if (($contact['avatar'] != $avatar) || empty($contact['blurhash'])) { $update_fields = ['avatar' => $avatar]; - if (!Network::isLocalLink($avatar) && Network::isValidHttpUrl($avatar)) { + if (!Network::isLocalLink($avatar)) { $fetchResult = HTTPSignature::fetchRaw($avatar, 0, [HttpClientOptions::ACCEPT_CONTENT => [HttpClientAccept::IMAGE]]); $img_str = $fetchResult->getBody(); diff --git a/src/Model/Item.php b/src/Model/Item.php index 05ff10273c..0dd025539a 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -3682,7 +3682,7 @@ class Item return is_numeric($hookData['item_id']) ? $hookData['item_id'] : 0; } - $fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri); + $fetched_uri = ActivityPub\Processor::fetchMissingActivity($uri, [], '', ActivityPub\Receiver::COMPLETION_MANUAL, $uid); if ($fetched_uri) { $item_id = self::searchByLink($fetched_uri, $uid); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index bf2f7bb8e3..f47a200e65 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -36,6 +36,7 @@ use Friendica\Object\Image; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; use Friendica\Security\Security; +use Friendica\Util\Network; use Friendica\Util\Proxy; use Friendica\Util\Strings; @@ -582,8 +583,13 @@ class Photo $photo_failure = false; + if (!Network::isValidHttpUrl($image_url)) { + Logger::warning('Invalid image url', ['image_url' => $image_url, 'uid' => $uid, 'cid' => $cid, 'callstack' => System::callstack(20)]); + return false; + } + $filename = basename($image_url); - if (!empty($image_url) && @parse_url($image_url, PHP_URL_HOST)) { + if (!empty($image_url)) { $ret = DI::httpClient()->get($image_url, HttpClientAccept::IMAGE); Logger::debug('Got picture', ['Content-Type' => $ret->getHeader('Content-Type'), 'url' => $image_url]); $img_str = $ret->getBody(); diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 897a39e60e..d466800594 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -194,7 +194,7 @@ class Tag } elseif (Contact::getIdForURL($url, 0, $fetch ? null : false)) { $target = self::ACCOUNT; Logger::debug('URL is an account', ['url' => $url]); - } elseif ($fetch && ($target != self::GENERAL_COLLECTION) && Network::isValidHttpUrl($url)) { + } elseif ($fetch && ($target != self::GENERAL_COLLECTION)) { $content = ActivityPub::fetchContent($url); if (!empty($content['type']) && ($content['type'] == 'OrderedCollection')) { $target = self::GENERAL_COLLECTION; diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 92e049ce39..00295bc2f6 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -121,7 +121,7 @@ class Probe $numeric_fields = ['gsid', 'hide', 'account-type', 'manually-approve']; if (!empty($data['photo']) && !Network::isValidHttpUrl($data['photo'])) { - Logger::info('Invalid URL for photo', ['url' => $data['url'], 'photo' => $data['photo']]); + Logger::warning('Invalid URL for photo', ['url' => $data['url'], 'photo' => $data['photo']]); unset($data['photo']); } diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index d57842c2b2..6836eafe34 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -1467,13 +1467,14 @@ class Processor * @param array $child activity array with the child of this message * @param string $relay_actor Relay actor * @param int $completion Completion mode, see Receiver::COMPLETION_* + * @param int $uid User id that is used to fetch the activity * @return string fetched message URL * @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \ImagickException */ - public static function fetchMissingActivity(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL): string + public static function fetchMissingActivity(string $url, array $child = [], string $relay_actor = '', int $completion = Receiver::COMPLETION_MANUAL, int $uid = 0): string { - $object = self::fetchCachedActivity($url, 0); + $object = self::fetchCachedActivity($url, $uid); if (empty($object)) { return ''; } @@ -1536,17 +1537,21 @@ class Processor Contact::updateByUrlIfNeeded($actor); - if (!empty($relay_actor)) { - $ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor); - $ldactivity['completion-mode'] = Receiver::COMPLETION_RELAY; - } elseif (!empty($child['thread-completion'])) { + if (!empty($child['thread-completion'])) { $ldactivity['thread-completion'] = $child['thread-completion']; $ldactivity['completion-mode'] = $child['completion-mode'] ?? Receiver::COMPLETION_NONE; } else { - $ldactivity['thread-completion'] = Contact::getIdForURL($actor); + $ldactivity['thread-completion'] = Contact::getIdForURL($relay_actor ?: $actor); $ldactivity['completion-mode'] = $completion; } + if ($completion == Receiver::COMPLETION_RELAY) { + $ldactivity['from-relay'] = $ldactivity['thread-completion']; + if (!self::acceptIncomingMessage($ldactivity, $object['id'])) { + return ''; + } + } + if (!empty($child['thread-children-type'])) { $ldactivity['thread-children-type'] = $child['thread-children-type']; } elseif (!empty($child['type'])) { @@ -1555,13 +1560,9 @@ class Processor $ldactivity['thread-children-type'] = 'as:Create'; } - if (!empty($relay_actor) && !self::acceptIncomingMessage($ldactivity, $object['id'])) { - return ''; - } - if (($completion == Receiver::COMPLETION_RELAY) && Queue::exists($url, 'as:Create')) { Logger::notice('Activity has already been queued.', ['url' => $url, 'object' => $activity['id']]); - } elseif (ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), 0, true, false, $signer, '', $completion)) { + } elseif (ActivityPub\Receiver::processActivity($ldactivity, json_encode($activity), $uid, true, false, $signer, '', $completion)) { Logger::notice('Activity had been fetched and processed.', ['url' => $url, 'entry' => $child['entry-id'] ?? 0, 'completion' => $completion, 'object' => $activity['id']]); } else { Logger::notice('Activity had been fetched and will be processed later.', ['url' => $url, 'entry' => $child['entry-id'] ?? 0, 'completion' => $completion, 'object' => $activity['id']]); diff --git a/src/Protocol/ActivityPub/Queue.php b/src/Protocol/ActivityPub/Queue.php index 25b431c1b0..9180de08f2 100644 --- a/src/Protocol/ActivityPub/Queue.php +++ b/src/Protocol/ActivityPub/Queue.php @@ -236,7 +236,7 @@ class Queue } DBA::close($receivers); - if (!Receiver::routeActivities($activity, $type, $push, $fetch_parents)) { + if (!Receiver::routeActivities($activity, $type, $push, $fetch_parents, $activity['receiver'][0] ?? 0)) { self::remove($activity); } diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index 6bcf97057a..6654b18fc6 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -415,7 +415,6 @@ class Receiver $object_data['object_id'] = JsonLD::fetchElement($activity, 'as:object', '@id'); $object_data['object_ids'] = JsonLD::fetchElementArray($activity, 'as:object', '@id'); $object_data['content'] = JsonLD::fetchElement($activity, 'as:content', '@type'); - $object_data['push'] = $push; } elseif (in_array($object_type, self::ACCOUNT_TYPES)) { $object_data = []; $object_data['id'] = JsonLD::fetchElement($activity, '@id'); @@ -423,16 +422,14 @@ class Receiver $object_data['object_actor'] = JsonLD::fetchElement($activity['as:object'], 'as:actor', '@id'); $object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object'); $object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type'); - $object_data['push'] = $push; if (!$trust_source && ($type == 'as:Delete')) { $apcontact = APContact::getByURL($object_data['object_id'], true); $trust_source = empty($apcontact) || ($apcontact['type'] == 'Tombstone') || $apcontact['suspended']; } - } elseif (in_array($type, ['as:Create', 'as:Update', 'as:Announce', 'as:Invite']) || strpos($type, '#emojiReaction')) { + } elseif (in_array($type, ['as:Create', 'as:Update', 'as:Invite']) || strpos($type, '#emojiReaction')) { // Fetch the content only on activities where this matters // We can receive "#emojiReaction" when fetching content from Hubzilla systems - // Always fetch on "Announce" - $object_data = self::fetchObject($object_id, $activity['as:object'], $trust_source && ($type != 'as:Announce'), $fetch_uid); + $object_data = self::fetchObject($object_id, $activity['as:object'], $trust_source, $fetch_uid); if (empty($object_data)) { Logger::info("Object data couldn't be processed"); return []; @@ -440,19 +437,13 @@ class Receiver $object_data['object_id'] = $object_id; - if ($type == 'as:Announce') { - $object_data['push'] = false; - } else { - $object_data['push'] = $push; - } - // Test if it is an answer to a mail if (DBA::exists('mail', ['uri' => $object_data['reply-to-id']])) { $object_data['directmessage'] = true; } else { $object_data['directmessage'] = JsonLD::fetchElement($activity, 'litepub:directMessage'); } - } elseif (in_array($type, array_merge(self::ACTIVITY_TYPES, ['as:Follow'])) && in_array($object_type, self::CONTENT_TYPES)) { + } elseif (in_array($type, array_merge(self::ACTIVITY_TYPES, ['as:Announce', 'as:Follow'])) && in_array($object_type, self::CONTENT_TYPES)) { // Create a mostly empty array out of the activity data (instead of the object). // This way we later don't have to check for the existence of each individual array element. $object_data = self::processObject($activity); @@ -460,7 +451,6 @@ class Receiver $object_data['author'] = JsonLD::fetchElement($activity, 'as:actor', '@id'); $object_data['object_id'] = $object_id; $object_data['object_type'] = ''; // Since we don't fetch the object, we don't know the type - $object_data['push'] = $push; } elseif (in_array($type, ['as:Add', 'as:Remove', 'as:Move'])) { $object_data = []; $object_data['id'] = JsonLD::fetchElement($activity, '@id'); @@ -468,7 +458,6 @@ class Receiver $object_data['object_id'] = JsonLD::fetchElement($activity, 'as:object', '@id'); $object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type'); $object_data['object_content'] = JsonLD::fetchElement($activity['as:object'], 'as:content', '@type'); - $object_data['push'] = $push; } else { $object_data = []; $object_data['id'] = JsonLD::fetchElement($activity, '@id'); @@ -476,7 +465,6 @@ class Receiver $object_data['object_actor'] = JsonLD::fetchElement($activity['as:object'], 'as:actor', '@id'); $object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object'); $object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type'); - $object_data['push'] = $push; // An Undo is done on the object of an object, so we need that type as well if (($type == 'as:Undo') && !empty($object_data['object_object'])) { @@ -491,6 +479,8 @@ class Receiver } } + $object_data['push'] = $push; + $object_data = self::addActivityFields($object_data, $activity); if (empty($object_data['object_type'])) { @@ -653,7 +643,9 @@ class Receiver } } - if (DI::config()->get('system', 'decoupled_receiver') && ($trust_source || DI::config()->get('debug', 'ap_inbox_store_untrusted'))) { + $decouple = DI::config()->get('system', 'decoupled_receiver') && !in_array($completion, [self::COMPLETION_MANUAL, self::COMPLETION_ANNOUCE]); + + if ($decouple && ($trust_source || DI::config()->get('debug', 'ap_inbox_store_untrusted'))) { $object_data = Queue::add($object_data, $type, $uid, $http_signer, $push, $trust_source); } @@ -662,7 +654,7 @@ class Receiver return true; } - if (!empty($object_data['entry-id']) && DI::config()->get('system', 'decoupled_receiver') && ($push || ($completion == self::COMPLETION_RELAY))) { + if (!empty($object_data['entry-id']) && $decouple && ($push || ($completion == self::COMPLETION_RELAY))) { if (Queue::isProcessable($object_data['entry-id'])) { // We delay by 5 seconds to allow to accumulate all receivers $delayed = date(DateTimeFormat::MYSQL, time() + 5); @@ -679,7 +671,7 @@ class Receiver $object_data['recursion-depth'] = $activity['recursion-depth']; } - if (!self::routeActivities($object_data, $type, $push)) { + if (!self::routeActivities($object_data, $type, $push, true, $uid)) { self::storeUnhandledActivity(true, $type, $object_data, $activity, $body, $uid, $trust_source, $push, $signer); Queue::remove($object_data); } @@ -693,10 +685,11 @@ class Receiver * @param string $type * @param bool $push * @param bool $fetch_parents + * @param int $uid * * @return boolean Could the activity be routed? */ - public static function routeActivities(array $object_data, string $type, bool $push, bool $fetch_parents = true): bool + public static function routeActivities(array $object_data, string $type, bool $push, bool $fetch_parents = true, int $uid = 0): bool { $activity = $object_data['object_activity'] ?? []; @@ -739,43 +732,24 @@ class Receiver case 'as:Announce': if (in_array($object_data['object_type'], self::CONTENT_TYPES)) { - $actor = JsonLD::fetchElement($activity, 'as:actor', '@id'); - $object_data['thread-completion'] = Contact::getIdForURL($actor); - $object_data['completion-mode'] = self::COMPLETION_ANNOUCE; - - if (!Post::exists(['uri' => $object_data['id'], 'uid' => 0])) { - $item = ActivityPub\Processor::createItem($object_data, $fetch_parents); - if (empty($item)) { - Logger::debug('announced id was not created', ['id' => $object_data['id']]); + if (!Item::searchByLink($object_data['object_id'], $uid)) { + if (ActivityPub\Processor::fetchMissingActivity($object_data['object_id'], [], $object_data['actor'], self::COMPLETION_ANNOUCE, $uid)) { + Logger::debug('Created announced id', ['uid' => $uid, 'id' => $object_data['object_id']]); + Queue::remove($object_data); + } else { + Logger::debug('Announced id was not created', ['uid' => $uid, 'id' => $object_data['object_id']]); Queue::remove($object_data); return true; } - - $item['post-reason'] = Item::PR_ANNOUNCEMENT; - ActivityPub\Processor::postItem($object_data, $item); - Logger::debug('Created announced id', ['id' => $object_data['id']]); } else { - Logger::info('Announced id already exists', ['id' => $object_data['id']]); + Logger::info('Announced id already exists', ['uid' => $uid, 'id' => $object_data['object_id']]); Queue::remove($object_data); } - if (!empty($activity)) { - $announce_object_data = self::processObject($activity); - $announce_object_data['name'] = $type; - $announce_object_data['author'] = $actor; - $announce_object_data['object_id'] = $object_data['id']; - $announce_object_data['object_type'] = $object_data['object_type']; - $announce_object_data['push'] = $push; - Logger::debug('Create announce activity', ['id' => $announce_object_data['id'], 'object_data' => $announce_object_data]); - - if (!empty($object_data['raw'])) { - $announce_object_data['raw'] = $object_data['raw']; - } - if (!empty($object_data['raw-object'])) { - $announce_object_data['raw-object'] = $object_data['raw-object']; - } - ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE); - } + ActivityPub\Processor::createActivity($object_data, Activity::ANNOUNCE); + } elseif (in_array($object_data['object_type'], ['as:Tombstone', ''])) { + // We don't have the object here or it is deleted. We ignore this activity. + Queue::remove($object_data); } else { return false; } @@ -1066,7 +1040,7 @@ class Receiver foreach ($receiver_list as $receiver) { if ($receiver == 'Public') { - Logger::notice('Not compacted public collection found', ['activity' => $activity, 'callstack' => System::callstack(20)]); + Logger::warning('Not compacted public collection found', ['activity' => $activity, 'callstack' => System::callstack(20)]); $receiver = ActivityPub::PUBLIC_COLLECTION; } if ($receiver == self::PUBLIC_COLLECTION) { @@ -1474,14 +1448,6 @@ class Receiver return $object_data; } - if ($type == 'as:Announce') { - $object_id = JsonLD::fetchElement($object, 'object', '@id'); - if (empty($object_id) || !is_string($object_id)) { - return false; - } - return self::fetchObject($object_id, [], false, $uid); - } - Logger::info('Unhandled object type: ' . $type); return false; } @@ -1899,7 +1865,7 @@ class Receiver $object_data['reply-to-id'] = $object_data['id']; // On activities the "reply to" is the id of the object it refers to - if (in_array($object_data['object_type'], self::ACTIVITY_TYPES)) { + if (in_array($object_data['object_type'], array_merge(self::ACTIVITY_TYPES, ['as:Announce']))) { $object_id = JsonLD::fetchElement($object, 'as:object', '@id'); if (!empty($object_id)) { $object_data['reply-to-id'] = $object_id; diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index bf03e2cf03..15a9c856da 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -159,7 +159,7 @@ class OnePoll } if (!Network::isValidHttpUrl($contact['poll'])) { - Logger::notice('Poll address is not valid', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'poll' => $contact['poll']]); + Logger::warning('Poll address is not valid', ['id' => $contact['id'], 'uid' => $contact['uid'], 'url' => $contact['url'], 'poll' => $contact['poll']]); return false; } diff --git a/src/Worker/PollContacts.php b/src/Worker/PollContacts.php index 9ae320f956..ba908cb8f3 100644 --- a/src/Worker/PollContacts.php +++ b/src/Worker/PollContacts.php @@ -41,7 +41,7 @@ class PollContacts $abandon_days = 0; } - $condition = ['network' => [Protocol::FEED, Protocol::MAIL, Protocol::OSTATUS], 'self' => false, 'blocked' => false]; + $condition = ['network' => [Protocol::FEED, Protocol::MAIL, Protocol::OSTATUS], 'self' => false, 'blocked' => false, 'archive' => false]; if (!empty($abandon_days)) { $condition = DBA::mergeConditions($condition,