Merge pull request #10508 from annando/loop-prevention

Prevent endless loop when updating contact by probe
This commit is contained in:
Hypolite Petovan 2021-07-19 10:21:10 -04:00 committed by GitHub
commit 7cbe1e3ca2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 137 additions and 76 deletions

View file

@ -31,11 +31,13 @@ use Friendica\DI;
use Friendica\Network\Probe; use Friendica\Network\Probe;
use Friendica\Protocol\ActivityNamespace; use Friendica\Protocol\ActivityNamespace;
use Friendica\Protocol\ActivityPub; use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\ActivityPub\Transmitter;
use Friendica\Util\Crypto; use Friendica\Util\Crypto;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\HTTPSignature; use Friendica\Util\HTTPSignature;
use Friendica\Util\JsonLD; use Friendica\Util\JsonLD;
use Friendica\Util\Network; use Friendica\Util\Network;
use Friendica\Util\Strings;
class APContact class APContact
{ {
@ -52,6 +54,20 @@ class APContact
return []; return [];
} }
if (Contact::isLocal($addr) && ($local_uid = User::getIdForURL($addr)) && ($local_owner = User::getOwnerDataById($local_uid))) {
$data = [
'addr' => $local_owner['addr'],
'baseurl' => $local_owner['baseurl'],
'url' => $local_owner['url'],
'subscribe' => $local_owner['baseurl'] . '/follow?url={uri}'];
if (!empty($local_owner['alias']) && ($local_owner['url'] != $local_owner['alias'])) {
$data['alias'] = $local_owner['alias'];
}
return $data;
}
$data = ['addr' => $addr]; $data = ['addr' => $addr];
$template = 'https://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr); $template = 'https://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr);
$webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), 'application/jrd+json'); $webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), 'application/jrd+json');
@ -148,39 +164,51 @@ class APContact
$url = $apcontact['url']; $url = $apcontact['url'];
} }
$curlResult = HTTPSignature::fetchRaw($url);
$failed = empty($curlResult) || empty($curlResult->getBody()) ||
(!$curlResult->isSuccess() && ($curlResult->getReturnCode() != 410));
if (!$failed) {
$data = json_decode($curlResult->getBody(), true);
$failed = empty($data) || !is_array($data);
}
if (!$failed && ($curlResult->getReturnCode() == 410)) {
$data = ['@context' => ActivityPub::CONTEXT, 'id' => $url, 'type' => 'Tombstone'];
}
if ($failed) {
self::markForArchival($fetched_contact ?: []);
return $fetched_contact;
}
$compacted = JsonLD::compact($data);
if (empty($compacted['@id'])) {
return $fetched_contact;
}
// Detect multiple fast repeating request to the same address // Detect multiple fast repeating request to the same address
// See https://github.com/friendica/friendica/issues/9303 // See https://github.com/friendica/friendica/issues/9303
$cachekey = 'apcontact:getByURL:' . $url; $cachekey = 'apcontact:getByURL:' . $url;
$result = DI::cache()->get($cachekey); $result = DI::cache()->get($cachekey);
if (!is_null($result)) { if (!is_null($result)) {
Logger::notice('Multiple requests for the address', ['url' => $url, 'update' => $update, 'callstack' => System::callstack(20), 'result' => $result]); Logger::notice('Multiple requests for the address', ['url' => $url, 'update' => $update, 'callstack' => System::callstack(20), 'result' => $result]);
if (!empty($fetched_contact)) {
return $fetched_contact;
}
} else { } else {
DI::cache()->set($cachekey, System::callstack(20), Duration::FIVE_MINUTES); DI::cache()->set($cachekey, System::callstack(20), Duration::FIVE_MINUTES);
} }
if (Network::isLocalLink($url) && ($local_uid = User::getIdForURL($url))) {
$data = Transmitter::getProfile($local_uid);
$local_owner = User::getOwnerDataById($local_uid);
}
if (empty($data)) {
$local_owner = [];
$curlResult = HTTPSignature::fetchRaw($url);
$failed = empty($curlResult) || empty($curlResult->getBody()) ||
(!$curlResult->isSuccess() && ($curlResult->getReturnCode() != 410));
if (!$failed) {
$data = json_decode($curlResult->getBody(), true);
$failed = empty($data) || !is_array($data);
}
if (!$failed && ($curlResult->getReturnCode() == 410)) {
$data = ['@context' => ActivityPub::CONTEXT, 'id' => $url, 'type' => 'Tombstone'];
}
if ($failed) {
self::markForArchival($fetched_contact ?: []);
return $fetched_contact;
}
}
$compacted = JsonLD::compact($data);
if (empty($compacted['@id'])) {
return $fetched_contact;
}
$apcontact['url'] = $compacted['@id']; $apcontact['url'] = $compacted['@id'];
$apcontact['uuid'] = JsonLD::fetchElement($compacted, 'diaspora:guid', '@value'); $apcontact['uuid'] = JsonLD::fetchElement($compacted, 'diaspora:guid', '@value');
$apcontact['type'] = str_replace('as:', '', JsonLD::fetchElement($compacted, '@type')); $apcontact['type'] = str_replace('as:', '', JsonLD::fetchElement($compacted, '@type'));
@ -264,7 +292,11 @@ class APContact
} }
if (!empty($apcontact['following'])) { if (!empty($apcontact['following'])) {
$following = ActivityPub::fetchContent($apcontact['following']); if (!empty($local_owner)) {
$following = ActivityPub\Transmitter::getContacts($local_owner, [Contact::SHARING, Contact::FRIEND], 'following');
} else {
$following = ActivityPub::fetchContent($apcontact['following']);
}
if (!empty($following['totalItems'])) { if (!empty($following['totalItems'])) {
// Mastodon seriously allows for this condition? // Mastodon seriously allows for this condition?
// Jul 14 2021 - See https://mastodon.social/@BLUW for a negative following count // Jul 14 2021 - See https://mastodon.social/@BLUW for a negative following count
@ -276,7 +308,11 @@ class APContact
} }
if (!empty($apcontact['followers'])) { if (!empty($apcontact['followers'])) {
$followers = ActivityPub::fetchContent($apcontact['followers']); if (!empty($local_owner)) {
$followers = ActivityPub\Transmitter::getContacts($local_owner, [Contact::FOLLOWER, Contact::FRIEND], 'followers');
} else {
$followers = ActivityPub::fetchContent($apcontact['followers']);
}
if (!empty($followers['totalItems'])) { if (!empty($followers['totalItems'])) {
// Mastodon seriously allows for this condition? // Mastodon seriously allows for this condition?
// Jul 14 2021 - See https://mastodon.online/@goes11 for a negative followers count // Jul 14 2021 - See https://mastodon.online/@goes11 for a negative followers count
@ -288,7 +324,11 @@ class APContact
} }
if (!empty($apcontact['outbox'])) { if (!empty($apcontact['outbox'])) {
$outbox = ActivityPub::fetchContent($apcontact['outbox']); if (!empty($local_owner)) {
$outbox = ActivityPub\Transmitter::getOutbox($local_owner);
} else {
$outbox = ActivityPub::fetchContent($apcontact['outbox']);
}
if (!empty($outbox['totalItems'])) { if (!empty($outbox['totalItems'])) {
$apcontact['statuses_count'] = $outbox['totalItems']; $apcontact['statuses_count'] = $outbox['totalItems'];
} }

View file

@ -453,6 +453,11 @@ class Contact
*/ */
public static function isLocal($url) public static function isLocal($url)
{ {
if (!parse_url($url, PHP_URL_SCHEME)) {
$addr_parts = explode('@', $url);
return (count($addr_parts) == 2) && ($addr_parts[1] == DI::baseUrl()->getHostname());
}
return Strings::compareLink(self::getBasepath($url, true), DI::baseUrl()); return Strings::compareLink(self::getBasepath($url, true), DI::baseUrl());
} }
@ -1808,7 +1813,7 @@ class Contact
// User contacts use are updated through the public contacts // User contacts use are updated through the public contacts
if (($uid != 0) && !in_array($contact['network'], [Protocol::FEED, Protocol::MAIL])) { if (($uid != 0) && !in_array($contact['network'], [Protocol::FEED, Protocol::MAIL])) {
$pcid = self::getIdForURL($contact['url'], false); $pcid = self::getIdForURL($contact['url'], 0, false);
if (!empty($pcid)) { if (!empty($pcid)) {
Logger::debug('Update the private contact via the public contact', ['id' => $cid, 'uid' => $uid, 'public' => $pcid]); Logger::debug('Update the private contact via the public contact', ['id' => $cid, 'uid' => $uid, 'public' => $pcid]);
self::updateAvatar($pcid, $avatar, $force, true); self::updateAvatar($pcid, $avatar, $force, true);
@ -2117,7 +2122,7 @@ class Contact
} }
if (Strings::normaliseLink($ret['url']) != Strings::normaliseLink($contact['url'])) { if (Strings::normaliseLink($ret['url']) != Strings::normaliseLink($contact['url'])) {
$cid = self::getIdForURL($ret['url']); $cid = self::getIdForURL($ret['url'], 0, false);
if (!empty($cid) && ($cid != $id)) { if (!empty($cid) && ($cid != $id)) {
Logger::notice('URL of contact changed.', ['id' => $id, 'new_id' => $cid, 'old' => $contact['url'], 'new' => $ret['url']]); Logger::notice('URL of contact changed.', ['id' => $id, 'new_id' => $cid, 'old' => $contact['url'], 'new' => $ret['url']]);
return self::updateFromProbeArray($cid, $ret); return self::updateFromProbeArray($cid, $ret);

View file

@ -804,30 +804,33 @@ class Photo
} }
/** /**
* Returns the GUID from picture links * Fetch the guid and scale from picture links
* *
* @param string $name Picture link * @param string $name Picture link
* @return string GUID * @return array
* @throws \Exception
*/ */
public static function getGUID($name) public static function getResourceData(string $name):array
{ {
$base = DI::baseUrl()->get(); $base = DI::baseUrl()->get();
$guid = str_replace([Strings::normaliseLink($base), '/photo/'], '', Strings::normaliseLink($name)); $guid = str_replace([Strings::normaliseLink($base), '/photo/'], '', Strings::normaliseLink($name));
if (parse_url($guid, PHP_URL_SCHEME)) {
return [];
}
$guid = self::stripExtension($guid); $guid = self::stripExtension($guid);
if (substr($guid, -2, 1) != "-") { if (substr($guid, -2, 1) != "-") {
return ''; return [];
} }
$scale = intval(substr($guid, -1, 1)); $scale = intval(substr($guid, -1, 1));
if (!is_numeric($scale)) { if (!is_numeric($scale)) {
return ''; return [];
} }
$guid = substr($guid, 0, -2); $guid = substr($guid, 0, -2);
return $guid; return ['guid' => $guid, 'scale' => $scale];
} }
/** /**
@ -839,13 +842,12 @@ class Photo
*/ */
public static function isLocal($name) public static function isLocal($name)
{ {
$guid = self::getGUID($name); $data = self::getResourceData($name);
if (empty($data)) {
if (empty($guid)) {
return false; return false;
} }
return DBA::exists('photo', ['resource-id' => $guid]); return DBA::exists('photo', ['resource-id' => $data['guid'], 'scale' => $data['scale']]);
} }
/** /**

View file

@ -74,6 +74,10 @@ class HTTPRequest implements IHTTPRequest
{ {
$stamp1 = microtime(true); $stamp1 = microtime(true);
if (Network::isLocalLink($url)) {
$this->logger->info('Local link', ['url' => $url, 'callstack' => System::callstack(20)]);
}
if (strlen($url) > 1000) { if (strlen($url) > 1000) {
$this->logger->debug('URL is longer than 1000 characters.', ['url' => $url, 'callstack' => System::callstack(20)]); $this->logger->debug('URL is longer than 1000 characters.', ['url' => $url, 'callstack' => System::callstack(20)]);
$this->profiler->saveTimestamp($stamp1, 'network'); $this->profiler->saveTimestamp($stamp1, 'network');
@ -226,6 +230,10 @@ class HTTPRequest implements IHTTPRequest
{ {
$stamp1 = microtime(true); $stamp1 = microtime(true);
if (Network::isLocalLink($url)) {
$this->logger->info('Local link', ['url' => $url, 'callstack' => System::callstack(20)]);
}
if (Network::isUrlBlocked($url)) { if (Network::isUrlBlocked($url)) {
$this->logger->info('Domain is blocked.' . ['url' => $url]); $this->logger->info('Domain is blocked.' . ['url' => $url]);
$this->profiler->saveTimestamp($stamp1, 'network'); $this->profiler->saveTimestamp($stamp1, 'network');
@ -328,6 +336,10 @@ class HTTPRequest implements IHTTPRequest
*/ */
public function finalUrl(string $url, int $depth = 1, bool $fetchbody = false) public function finalUrl(string $url, int $depth = 1, bool $fetchbody = false)
{ {
if (Network::isLocalLink($url)) {
$this->logger->info('Local link', ['url' => $url, 'callstack' => System::callstack(20)]);
}
if (Network::isUrlBlocked($url)) { if (Network::isUrlBlocked($url)) {
$this->logger->info('Domain is blocked.', ['url' => $url]); $this->logger->info('Domain is blocked.', ['url' => $url]);
return $url; return $url;

View file

@ -333,7 +333,7 @@ class Probe
public static function uri($uri, $network = '', $uid = -1) public static function uri($uri, $network = '', $uid = -1)
{ {
// Local profiles aren't probed via network // Local profiles aren't probed via network
if (empty($network) && strpos($uri, DI::baseUrl()->getHostname())) { if (empty($network) && Contact::isLocal($uri)) {
$data = self::localProbe($uri); $data = self::localProbe($uri);
if (!empty($data)) { if (!empty($data)) {
return $data; return $data;
@ -2201,39 +2201,33 @@ class Probe
*/ */
private static function localProbe(string $url) private static function localProbe(string $url)
{ {
$uid = User::getIdForURL($url); if ($uid = User::getIdForURL($url)) {
if (empty($uid)) { $profile = User::getOwnerDataById($uid);
return []; $approfile = ActivityPub\Transmitter::getProfile($uid);
}
$profile = User::getOwnerDataById($uid); if (empty($profile['gsid'])) {
if (empty($profile)) { $profile['gsid'] = GServer::getID($approfile['generator']['url']);
return []; }
}
$approfile = ActivityPub\Transmitter::getProfile($uid); $data = ['name' => $profile['name'], 'nick' => $profile['nick'], 'guid' => $approfile['diaspora:guid'] ?? '',
if (empty($approfile)) { 'url' => $profile['url'], 'addr' => $profile['addr'], 'alias' => $profile['alias'],
return []; 'photo' => Contact::getAvatarUrlForId($profile['id'], $profile['updated']),
'header' => $profile['header'] ? Contact::getHeaderUrlForId($profile['id'], $profile['updated']) : '',
'account-type' => $profile['contact-type'], 'community' => ($profile['contact-type'] == User::ACCOUNT_TYPE_COMMUNITY),
'keywords' => $profile['keywords'], 'location' => $profile['location'], 'about' => $profile['about'],
'hide' => !$profile['net-publish'], 'batch' => '', 'notify' => $profile['notify'],
'poll' => $profile['poll'], 'request' => $profile['request'], 'confirm' => $profile['confirm'],
'subscribe' => $approfile['generator']['url'] . '/follow?url={uri}', 'poco' => $profile['poco'],
'following' => $approfile['following'], 'followers' => $approfile['followers'],
'inbox' => $approfile['inbox'], 'outbox' => $approfile['outbox'],
'sharedinbox' => $approfile['endpoints']['sharedInbox'], 'network' => Protocol::DFRN,
'pubkey' => $profile['upubkey'], 'baseurl' => $approfile['generator']['url'], 'gsid' => $profile['gsid'],
'manually-approve' => in_array($profile['page-flags'], [User::PAGE_FLAGS_NORMAL, User::PAGE_FLAGS_PRVGROUP])];
} else {
// Default values for non existing targets
$data = ['name' => $url, 'nick' => $url, 'url' => $url, 'network' => Protocol::PHANTOM,
'photo' => DI::baseUrl() . Contact::DEFAULT_AVATAR_PHOTO];
} }
if (empty($profile['gsid'])) {
$profile['gsid'] = GServer::getID($approfile['generator']['url']);
}
$data = ['name' => $profile['name'], 'nick' => $profile['nick'], 'guid' => $approfile['diaspora:guid'] ?? '',
'url' => $profile['url'], 'addr' => $profile['addr'], 'alias' => $profile['alias'],
'photo' => Contact::getAvatarUrlForId($profile['id'], $profile['updated']),
'header' => $profile['header'] ? Contact::getHeaderUrlForId($profile['id'], $profile['updated']) : '',
'account-type' => $profile['contact-type'], 'community' => ($profile['contact-type'] == User::ACCOUNT_TYPE_COMMUNITY),
'keywords' => $profile['keywords'], 'location' => $profile['location'], 'about' => $profile['about'],
'hide' => !$profile['net-publish'], 'batch' => '', 'notify' => $profile['notify'],
'poll' => $profile['poll'], 'request' => $profile['request'], 'confirm' => $profile['confirm'],
'subscribe' => $approfile['generator']['url'] . '/follow?url={uri}', 'poco' => $profile['poco'],
'following' => $approfile['following'], 'followers' => $approfile['followers'],
'inbox' => $approfile['inbox'], 'outbox' => $approfile['outbox'],
'sharedinbox' => $approfile['endpoints']['sharedInbox'], 'network' => Protocol::DFRN,
'pubkey' => $profile['upubkey'], 'baseurl' => $approfile['generator']['url'], 'gsid' => $profile['gsid'],
'manually-approve' => in_array($profile['page-flags'], [User::PAGE_FLAGS_NORMAL, User::PAGE_FLAGS_PRVGROUP])];
return self::rearrangeData($data); return self::rearrangeData($data);
} }
} }

View file

@ -22,8 +22,6 @@
namespace Friendica\Protocol; namespace Friendica\Protocol;
use Friendica\Core\Protocol; use Friendica\Core\Protocol;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\APContact; use Friendica\Model\APContact;
use Friendica\Model\User; use Friendica\Model\User;
use Friendica\Util\HTTPSignature; use Friendica\Util\HTTPSignature;

View file

@ -22,8 +22,8 @@
namespace Friendica\Util; namespace Friendica\Util;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\System;
use Friendica\DI; use Friendica\DI;
use Friendica\Model\Photo;
/** /**
* Image utilities * Image utilities
@ -184,7 +184,17 @@ class Images
return $data; return $data;
} }
$img_str = DI::httpRequest()->fetch($url, 4); if (Network::isLocalLink($url) && ($data = Photo::getResourceData($url))) {
$photo = Photo::selectFirst([], ['resource-id' => $data['guid'], 'scale' => $data['scale']]);
if (!empty($photo)) {
$img_str = Photo::getImageDataForPhoto($photo);
}
// @todo Possibly add a check for locally stored files
}
if (empty($img_str)) {
$img_str = DI::httpRequest()->fetch($url, 4);
}
if (!$img_str) { if (!$img_str) {
return []; return [];