diff --git a/database.sql b/database.sql index deaa39a5b..ea89847c1 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.09-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1357 +-- DB_UPDATE_VERSION 1358 -- ------------------------------------------ @@ -102,6 +102,7 @@ CREATE TABLE IF NOT EXISTS `contact` ( `avatar-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `term-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `last-item` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'date of the last post', + `last-discovery` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'date of the last follower discovery', `priority` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', `blocked` boolean NOT NULL DEFAULT '1' COMMENT 'Node-wide block status', `block_reason` text COMMENT 'Node-wide block reason', @@ -342,8 +343,12 @@ CREATE TABLE IF NOT EXISTS `contact-relation` ( `cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'contact the related contact had interacted with', `relation-cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'related contact who had interacted with the contact', `last-interaction` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last interaction', + `follow-updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last update of the contact relationship', + `follows` boolean NOT NULL DEFAULT '0' COMMENT '', PRIMARY KEY(`cid`,`relation-cid`), - INDEX `relation-cid` (`relation-cid`) + INDEX `relation-cid` (`relation-cid`), + FOREIGN KEY (`cid`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE, + FOREIGN KEY (`relation-cid`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Contact relations'; -- @@ -517,17 +522,6 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts'; --- --- TABLE gfollower --- -CREATE TABLE IF NOT EXISTS `gfollower` ( - `gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact', - `follower-gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact of the follower', - `deleted` boolean NOT NULL DEFAULT '0' COMMENT '1 indicates that the connection has been deleted', - PRIMARY KEY(`gcid`,`follower-gcid`), - INDEX `follower-gcid` (`follower-gcid`) -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Followers of global contacts'; - -- -- TABLE glink -- diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 708c375b0..1cf8c7423 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1418,7 +1418,6 @@ class Contact 'poll' => $data['poll'] ?? '', 'name' => $data['name'] ?? '', 'nick' => $data['nick'] ?? '', - 'photo' => $data['photo'] ?? '', 'keywords' => $data['keywords'] ?? '', 'location' => $data['location'] ?? '', 'about' => $data['about'] ?? '', @@ -1474,14 +1473,6 @@ class Contact } else { // Else do a direct update self::updateFromProbe($contact_id, '', false); - - // Update the gcontact entry - if ($uid == 0) { - GContact::updateFromPublicContactID($contact_id); - if (($data['network'] == Protocol::ACTIVITYPUB) && in_array(DI::config()->get('system', 'gcontact_discovery'), [GContact::DISCOVERY_DIRECT, GContact::DISCOVERY_RECURSIVE])) { - GContact::discoverFollowers($data['url']); - } - } } } else { $fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'baseurl', 'gsid']; @@ -1818,9 +1809,11 @@ class Contact $uid = $contact['uid']; // Only update the cached photo links of public contacts when they already are cached - if (($uid == 0) && !$force && empty($contact['photo']) && empty($contact['thumb']) && empty($contact['micro'])) { - DBA::update('contact', ['avatar' => $avatar], ['id' => $cid]); - Logger::info('Only update the avatar', ['id' => $cid, 'avatar' => $avatar, 'contact' => $contact]); + if (($uid == 0) && !$force && empty($contact['thumb']) && empty($contact['micro'])) { + if ($contact['avatar'] != $avatar) { + DBA::update('contact', ['avatar' => $avatar], ['id' => $cid]); + Logger::info('Only update the avatar', ['id' => $cid, 'avatar' => $avatar, 'contact' => $contact]); + } return; } @@ -1830,28 +1823,27 @@ class Contact $contact['micro'] ?? '', ]; - foreach ($data as $image_uri) { - $image_rid = Photo::ridFromURI($image_uri); - if ($image_rid && !Photo::exists(['resource-id' => $image_rid, 'uid' => $uid])) { - Logger::info('Regenerating avatar', ['contact uid' => $uid, 'cid' => $cid, 'missing photo' => $image_rid, 'avatar' => $contact['avatar']]); - $force = true; + $update = ($contact['avatar'] != $avatar) || $force; + + if (!$update) { + foreach ($data as $image_uri) { + $image_rid = Photo::ridFromURI($image_uri); + if ($image_rid && !Photo::exists(['resource-id' => $image_rid, 'uid' => $uid])) { + Logger::info('Regenerating avatar', ['contact uid' => $uid, 'cid' => $cid, 'missing photo' => $image_rid, 'avatar' => $contact['avatar']]); + $update = true; + } } } - if (($contact["avatar"] != $avatar) || $force) { + if ($update) { $photos = Photo::importProfilePhoto($avatar, $uid, $cid, true); - if ($photos) { $fields = ['avatar' => $avatar, 'photo' => $photos[0], 'thumb' => $photos[1], 'micro' => $photos[2], 'avatar-date' => DateTimeFormat::utcNow()]; DBA::update('contact', $fields, ['id' => $cid]); - - // Update the public contact (contact id = 0) - if ($uid != 0) { - $pcontact = DBA::selectFirst('contact', ['id'], ['nurl' => $contact['nurl'], 'uid' => 0]); - if (DBA::isResult($pcontact)) { - DBA::update('contact', $fields, ['id' => $pcontact['id']]); - } - } + } elseif (empty($contact['avatar'])) { + // Ensure that the avatar field is set + DBA::update('contact', ['avatar' => $avatar], ['id' => $cid]); + Logger::info('Failed profile import', ['id' => $cid, 'force' => $force, 'avatar' => $avatar, 'contact' => $contact]); } } } @@ -2035,6 +2027,13 @@ class Contact $new_pubkey = $ret['pubkey']; + // Update the gcontact entry + if ($uid == 0) { + GContact::updateFromPublicContactID($id); + } + + ContactRelation::discoverByUrl($ret['url']); + $update = false; // make sure to not overwrite existing values with blank entries except some technical fields @@ -2508,7 +2507,6 @@ class Contact 'nurl' => Strings::normaliseLink($url), 'name' => $name, 'nick' => $nick, - 'photo' => $photo, 'network' => $network, 'rel' => self::FOLLOWER, 'blocked' => 0, diff --git a/src/Model/ContactRelation.php b/src/Model/ContactRelation.php new file mode 100644 index 000000000..06c25059f --- /dev/null +++ b/src/Model/ContactRelation.php @@ -0,0 +1,164 @@ +. + * + */ + +namespace Friendica\Model; + +use Friendica\Core\Logger; +use Friendica\Database\DBA; +use Friendica\DI; +use Friendica\Protocol\ActivityPub; +use Friendica\Util\DateTimeFormat; +use Friendica\Util\Strings; + +class ContactRelation +{ + /** + * No discovery of followers/followings + */ + const DISCOVERY_NONE = 0; + /** + * Discover followers/followings of local contacts + */ + const DISCOVERY_LOCAL = 1; + /** + * Discover followers/followings of local contacts and contacts that visibly interacted on the system + */ + const DISCOVERY_INTERACTOR = 2; + /** + * Discover followers/followings of all contacts + */ + const DISCOVERY_ALL = 3; + + public static function store(int $target, int $actor, string $interaction_date) + { + if ($actor == $target) { + return; + } + + DBA::update('contact-relation', ['last-interaction' => $interaction_date], ['cid' => $target, 'relation-cid' => $actor], true); + } + + /** + * Fetches the followers of a given profile and adds them + * + * @param string $url URL of a profile + * @return void + */ + public static function discoverByUrl(string $url) + { + $contact_discovery = DI::config()->get('system', 'contact_discovery'); + + if ($contact_discovery == self::DISCOVERY_NONE) { + return; + } + + $contact = Contact::getByURL($url); + if (empty($contact)) { + return; + } + + if ($contact['last-discovery'] > DateTimeFormat::utc('now - 1 month')) { + Logger::info('No discovery - Last was less than a month ago.', ['id' => $contact['id'], 'url' => $url, 'discovery' => $contact['last-discovery']]); + return; + } + + if ($contact_discovery != self::DISCOVERY_ALL) { + $local = DBA::exists('contact', ["`nurl` = ? AND `uid` != ?", Strings::normaliseLink($url), 0]); + if (($contact_discovery == self::DISCOVERY_LOCAL) && !$local) { + Logger::info('No discovery - This contact is not followed/following locally.', ['id' => $contact['id'], 'url' => $url]); + return; + } + + if ($contact_discovery == self::DISCOVERY_INTERACTOR) { + $interactor = DBA::exists('contact-relation', ["`relation-cid` = ? AND `last-interaction` > ?", $contact['id'], DBA::NULL_DATETIME]); + if (!$local && !$interactor) { + Logger::info('No discovery - This contact is not interacting locally.', ['id' => $contact['id'], 'url' => $url]); + return; + } + } + } elseif ($contact['created'] > DateTimeFormat::utc('now - 1 day')) { + // Newly created contacts are not discovered to avoid DDoS attacks + Logger::info('No discovery - Contact record is less than a day old.', ['id' => $contact['id'], 'url' => $url, 'discovery' => $contact['created']]); + return; + } + + $apcontact = APContact::getByURL($url); + + if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { + $followers = ActivityPub::fetchItems($apcontact['followers']); + } else { + $followers = []; + } + + if (!empty($apcontact['following']) && is_string($apcontact['following'])) { + $followings = ActivityPub::fetchItems($apcontact['following']); + } else { + $followings = []; + } + + if (empty($followers) && empty($followings)) { + return; + } + + $target = $contact['id']; + + if (!empty($followers)) { + // Clear the follower list, since it will be recreated in the next step + DBA::update('contact-relation', ['follows' => false], ['cid' => $target]); + } + + $contacts = []; + foreach (array_merge($followers, $followings) as $contact) { + if (is_string($contact)) { + $contacts[] = $contact; + } elseif (!empty($contact['url']) && is_string($contact['url'])) { + $contacts[] = $contact['url']; + } + } + $contacts = array_unique($contacts); + + Logger::info('Discover contacts', ['id' => $target, 'url' => $url, 'contacts' => count($contacts)]); + foreach ($contacts as $contact) { + $actor = Contact::getIdForURL($contact); + if (!empty($actor)) { + $fields = []; + if (in_array($contact, $followers)) { + $fields = ['cid' => $target, 'relation-cid' => $actor]; + } elseif (in_array($contact, $followings)) { + $fields = ['cid' => $actor, 'relation-cid' => $target]; + } else { + continue; + } + + DBA::update('contact-relation', ['follows' => true, 'follow-updated' => DateTimeFormat::utcNow()], $fields, true); + } + } + + if (!empty($followers)) { + // Delete all followers that aren't followers anymore (and aren't interacting) + DBA::delete('contact-relation', ['cid' => $target, 'follows' => false, 'last-interaction' => DBA::NULL_DATETIME]); + } + + DBA::update('contact', ['last-discovery' => DateTimeFormat::utcNow()], ['id' => $target]); + Logger::info('Contacts discovery finished, "last-discovery" set', ['id' => $target, 'url' => $url]); + return; + } +} diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 41ca763fc..ec53133c9 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -43,19 +43,6 @@ use Friendica\Util\Strings; */ class GContact { - /** - * No discovery of followers/followings - */ - const DISCOVERY_NONE = 0; - /** - * Only discover followers/followings from direct contacts - */ - const DISCOVERY_DIRECT = 1; - /** - * Recursive discovery of followers/followings - */ - const DISCOVERY_RECURSIVE = 2; - /** * Search global contact table by nick or name * @@ -1288,129 +1275,6 @@ class GContact } } - /** - * Fetches the followers of a given profile and adds them - * - * @param string $url URL of a profile - * @return void - */ - public static function discoverFollowers(string $url) - { - $gcontact = DBA::selectFirst('gcontact', ['id', 'last_discovery'], ['nurl' => Strings::normaliseLink(($url))]); - if (!DBA::isResult($gcontact)) { - return; - } - - if ($gcontact['last_discovery'] > DateTimeFormat::utc('now - 1 month')) { - Logger::info('Last discovery was less then a month before.', ['url' => $url, 'discovery' => $gcontact['last_discovery']]); - return; - } - - $gcid = $gcontact['id']; - - $apcontact = APContact::getByURL($url); - - if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { - $followers = ActivityPub::fetchItems($apcontact['followers']); - } else { - $followers = []; - } - - if (!empty($apcontact['following']) && is_string($apcontact['following'])) { - $followings = ActivityPub::fetchItems($apcontact['following']); - } else { - $followings = []; - } - - if (!empty($followers) || !empty($followings)) { - if (!empty($followers)) { - // Clear the follower list, since it will be recreated in the next step - DBA::update('gfollower', ['deleted' => true], ['gcid' => $gcid]); - } - - $contacts = []; - foreach (array_merge($followers, $followings) as $contact) { - if (is_string($contact)) { - $contacts[] = $contact; - } elseif (!empty($contact['url']) && is_string($contact['url'])) { - $contacts[] = $contact['url']; - } - } - $contacts = array_unique($contacts); - - Logger::info('Discover AP contacts', ['url' => $url, 'contacts' => count($contacts)]); - foreach ($contacts as $contact) { - $gcontact = DBA::selectFirst('gcontact', ['id'], ['nurl' => Strings::normaliseLink(($contact))]); - if (DBA::isResult($gcontact)) { - $fields = []; - if (in_array($contact, $followers)) { - $fields = ['gcid' => $gcid, 'follower-gcid' => $gcontact['id']]; - } elseif (in_array($contact, $followings)) { - $fields = ['gcid' => $gcontact['id'], 'follower-gcid' => $gcid]; - } - - if (!empty($fields)) { - Logger::info('Set relation between contacts', $fields); - DBA::update('gfollower', ['deleted' => false], $fields, true); - continue; - } - } - - if (!Network::isUrlBlocked($contact)) { - Logger::info('Discover new AP contact', ['url' => $contact]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact, 'nodiscover'); - } else { - Logger::info('No discovery, the URL is blocked.', ['url' => $contact]); - } - } - if (!empty($followers)) { - // Delete all followers that aren't undeleted - DBA::delete('gfollower', ['gcid' => $gcid, 'deleted' => true]); - } - - DBA::update('gcontact', ['last_discovery' => DateTimeFormat::utcNow()], ['id' => $gcid]); - Logger::info('AP contacts discovery finished, last discovery set', ['url' => $url]); - return; - } - - $data = Probe::uri($url); - if (empty($data['poco'])) { - return; - } - - $curlResult = DI::httpRequest()->get($data['poco']); - if (!$curlResult->isSuccess()) { - return; - } - $poco = json_decode($curlResult->getBody(), true); - if (empty($poco['entry'])) { - return; - } - - Logger::info('PoCo Discovery started', ['url' => $url, 'contacts' => count($poco['entry'])]); - - foreach ($poco['entry'] as $entries) { - if (!empty($entries['urls'])) { - foreach ($entries['urls'] as $entry) { - if ($entry['type'] == 'profile') { - if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($entry['value']))])) { - continue; - } - if (!Network::isUrlBlocked($entry['value'])) { - Logger::info('Discover new PoCo contact', ['url' => $entry['value']]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $entry['value'], 'nodiscover'); - } else { - Logger::info('No discovery, the URL is blocked.', ['url' => $entry['value']]); - } - } - } - } - } - - DBA::update('gcontact', ['last_discovery' => DateTimeFormat::utcNow()], ['id' => $gcid]); - Logger::info('PoCo Discovery finished', ['url' => $url]); - } - /** * Returns a random, global contact of the current node * diff --git a/src/Model/Item.php b/src/Model/Item.php index 1a561c7f1..d8fbac830 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1554,9 +1554,7 @@ class Item } // Update the contact relations - if ($item['author-id'] != $parent['author-id']) { - DBA::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true); - } + ContactRelation::store($parent['author-id'], $item['author-id'], $item['created']); } return $item; diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index 290d92d13..c1d4479d9 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -28,7 +28,7 @@ use Friendica\Core\Theme; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Model\GContact; +use Friendica\Model\ContactRelation; use Friendica\Module\BaseAdmin; use Friendica\Module\Register; use Friendica\Protocol\PortableContact; @@ -178,8 +178,8 @@ class Site extends BaseAdmin $min_memory = (!empty($_POST['min_memory']) ? intval(trim($_POST['min_memory'])) : 0); $optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100); $optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30); + $contact_discovery = (!empty($_POST['contact_discovery']) ? intval(trim($_POST['contact_discovery'])) : ContactRelation::DISCOVERY_NONE); $poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false); - $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : GContact::DISCOVERY_NONE); $poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7); $poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED); $poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30); @@ -308,7 +308,7 @@ class Site extends BaseAdmin DI::config()->set('system', 'optimize_max_tablesize', $optimize_max_tablesize); DI::config()->set('system', 'optimize_fragmentation', $optimize_fragmentation); DI::config()->set('system', 'poco_completion' , $poco_completion); - DI::config()->set('system', 'gcontact_discovery' , $gcontact_discovery); + DI::config()->set('system', 'contact_discovery' , $contact_discovery); DI::config()->set('system', 'poco_requery_days' , $poco_requery_days); DI::config()->set('system', 'poco_discovery' , $poco_discovery); DI::config()->set('system', 'poco_discovery_since' , $poco_discovery_since); @@ -551,9 +551,11 @@ class Site extends BaseAdmin ]; $discovery_choices = [ - GContact::DISCOVERY_NONE => DI::l10n()->t('none'), - GContact::DISCOVERY_DIRECT => DI::l10n()->t('Direct contacts'), - GContact::DISCOVERY_RECURSIVE => DI::l10n()->t('Contacts of contacts') + ContactRelation::DISCOVERY_NONE => DI::l10n()->t('none'), + ContactRelation::DISCOVERY_LOCAL => DI::l10n()->t('Local contacts'), + ContactRelation::DISCOVERY_INTERACTOR => DI::l10n()->t('Interactors'), + // "All" is deactivated until we are sure not to put too much stress on the fediverse with this + // ContactRelation::DISCOVERY_ALL => DI::l10n()->t('All'), ]; $diaspora_able = (DI::baseUrl()->getUrlPath() == ''); @@ -677,8 +679,13 @@ class Site extends BaseAdmin '$optimize_max_tablesize' => ['optimize_max_tablesize', DI::l10n()->t('Maximum table size for optimization'), $optimize_max_tablesize, DI::l10n()->t('Maximum table size (in MB) for the automatic optimization. Enter -1 to disable it.')], '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], + '$contact_discovery' => ['contact_discovery', DI::l10n()->t('Discover followers/followings from contacts'), DI::config()->get('system', 'contact_discovery'), DI::l10n()->t('If enabled, contacts are checked for their followers and following contacts.') . '