diff --git a/database.sql b/database.sql index 3cb87fc90..b4c65fb89 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.03-dev (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1335 +-- DB_UPDATE_VERSION 1336 -- ------------------------------------------ @@ -393,6 +393,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `updated` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', + `last_discovery` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last contact discovery', `archive_date` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `archived` boolean NOT NULL DEFAULT '0' COMMENT '', `location` varchar(255) NOT NULL DEFAULT '' COMMENT '', @@ -419,6 +420,17 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( INDEX `updated` (`updated`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts'; +-- +-- TABLE gfollower +-- +CREATE TABLE IF NOT EXISTS `gfollower` ( + `gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact', + `follower-gcid` int unsigned NOT NULL DEFAULT 0 COMMENT 'global contact of the follower', + `deleted` boolean NOT NULL DEFAULT '0' COMMENT '1 indicates that the connection has been deleted', + PRIMARY KEY(`gcid`,`follower-gcid`), + INDEX `follower-gcid` (`follower-gcid`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Followers of global contacts'; + -- -- TABLE glink -- diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 95afc1be6..a13b719a9 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -28,6 +28,7 @@ use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\System; use Friendica\Core\Search; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Network\Probe; @@ -1275,6 +1276,136 @@ class GContact } } + /** + * Fetches the followers of a given profile and adds them + * + * @param string $url URL of a profile + * @return void + */ + public static function discoverFollowers(string $url, int $following_gcid = 0, int $follower_gcid = 0) + { + $gcontact = DBA::selectFirst('gcontact', ['id', 'last_discovery'], ['nurl' => Strings::normaliseLink(($url))]); + if (!DBA::isResult($gcontact)) { + return; + } + + if ($gcontact['last_discovery'] > DateTimeFormat::utc('now - 1 month')) { + Logger::info('Last discovery was less then a month before.', ['url' => $url, 'discovery' => $gcontact['last_discovery']]); + return; + } + + $gcid = $gcontact['id']; + + if (!empty($following_gcid)) { + $fields = ['gcid' => $following_gcid, 'follower-gcid' => $gcid]; + Logger::info('Set relation for followed gcontact', $fields); + DBA::update('gfollower', ['deleted' => false], $fields, true); + } elseif (!empty($follower_gcid)) { + $fields = ['gcid' => $gcid, 'follower-gcid' => $follower_gcid]; + Logger::info('Set relation for following gcontact', $fields); + DBA::update('gfollower', ['deleted' => false], $fields, true); + } + + $apcontact = APContact::getByURL($url); + + if (!empty($apcontact['followers']) && is_string($apcontact['followers'])) { + $followers = ActivityPub::fetchItems($apcontact['followers']); + } else { + $followers = []; + } + + if (!empty($apcontact['following']) && is_string($apcontact['following'])) { + $followings = ActivityPub::fetchItems($apcontact['following']); + } else { + $followings = []; + } + + if (!empty($followers) || !empty($followings)) { + if (!empty($followers)) { + // Clear the follower list, since it will be recreated in the next step + DBA::update('gfollower', ['deleted' => true], ['gcid' => $gcid]); + } + + $contacts = []; + foreach (array_merge($followers, $followings) as $contact) { + if (is_string($contact)) { + $contacts[] = $contact; + } elseif (!empty($contact['url']) && is_string($contact['url'])) { + $contacts[] = $contact['url']; + } + } + $contacts = array_unique($contacts); + + Logger::info('Discover AP contacts', ['url' => $url, 'contacts' => count($contacts)]); + foreach ($contacts as $contact) { + $gcontact = DBA::selectFirst('gcontact', ['id'], ['nurl' => Strings::normaliseLink(($contact))]); + if (DBA::isResult($gcontact)) { + if (in_array($contact, $followers)) { + $fields = ['gcid' => $gcid, 'follower-gcid' => $gcontact['id']]; + } elseif (in_array($contact, $followings)) { + $fields = ['gcid' => $gcontact['id'], 'follower-gcid' => $gcid]; + } + Logger::info('Set relation between contacts', $fields); + DBA::update('gfollower', ['deleted' => false], $fields, true); + continue; + } + + $follower_gcid = 0; + $following_gcid = 0; + + if (in_array($contact, $followers)) { + $following_gcid = $gcid; + } elseif (in_array($contact, $followings)) { + $follower_gcid = $gcid; + } + + Logger::info('Discover new AP contact', ['url' => $contact]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact, '', $following_gcid, $follower_gcid); + } + if (!empty($followers)) { + // Delete all followers that aren't undeleted + DBA::delete('gfollower', ['gcid' => $gcid, 'deleted' => true]); + } + + DBA::update('gcontact', ['last_discovery' => DateTimeFormat::utcNow()], ['id' => $gcid]); + Logger::info('AP contacts discovery finished, last discovery set', ['url' => $url]); + return; + } + + $data = Probe::uri($url); + if (empty($data['poco'])) { + return; + } + + $curlResult = Network::curl($data['poco']); + if (!$curlResult->isSuccess()) { + return; + } + $poco = json_decode($curlResult->getBody(), true); + if (empty($poco['entry'])) { + return; + } + + Logger::info('PoCo Discovery started', ['url' => $url, 'contacts' => count($poco['entry'])]); + + foreach ($poco['entry'] as $entries) { + if (!empty($entries['urls'])) { + foreach ($entries['urls'] as $entry) { + if ($entry['type'] == 'profile') { + if (DBA::exists('gcontact', ['nurl' => Strings::normaliseLink(($entry['value']))])) { + continue; + } + Logger::info('Discover new PoCo contact', ['url' => $entry['value']]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $entry['value']); + } + } + } + } + + DBA::update('gcontact', ['last_discovery' => DateTimeFormat::utcNow()], ['id' => $gcid]); + Logger::info('PoCo Discovery finished', ['url' => $url]); + } + /** * Returns a random, global contact of the current node * diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index ec6f01afa..846280c4f 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -178,6 +178,7 @@ class Site extends BaseAdmin $optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100); $optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30); $poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false); + $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : false); $poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7); $poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED); $poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30); @@ -305,6 +306,7 @@ class Site extends BaseAdmin DI::config()->set('system', 'optimize_max_tablesize', $optimize_max_tablesize); DI::config()->set('system', 'optimize_fragmentation', $optimize_fragmentation); DI::config()->set('system', 'poco_completion' , $poco_completion); + DI::config()->set('system', 'gcontact_discovery' , $gcontact_discovery); DI::config()->set('system', 'poco_requery_days' , $poco_requery_days); DI::config()->set('system', 'poco_discovery' , $poco_discovery); DI::config()->set('system', 'poco_discovery_since' , $poco_discovery_since); @@ -669,6 +671,7 @@ class Site extends BaseAdmin '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], '$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')], + '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.')], '$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')], '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index f1cd652f4..894c7f6d3 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -191,7 +191,7 @@ class ActivityPub */ public static function fetchOutbox($url, $uid) { - $data = self::fetchContent($url); + $data = self::fetchContent($url, $uid); if (empty($data)) { return; } @@ -213,6 +213,37 @@ class ActivityPub } } + /** + * Fetch items from AP endpoints + * + * @param string $url Address of the endpoint + * @param integer $uid Optional user id + * @return array Endpoint items + */ + public static function fetchItems(string $url, int $uid = 0) + { + $data = self::fetchContent($url, $uid); + if (empty($data)) { + return []; + } + + if (!empty($data['orderedItems'])) { + $items = $data['orderedItems']; + } elseif (!empty($data['first']['orderedItems'])) { + $items = $data['first']['orderedItems']; + } elseif (!empty($data['first']) && is_string($data['first'])) { + return self::fetchItems($data['first'], $uid); + } else { + $items = []; + } + + if (!empty($data['next']) && is_string($data['next'])) { + $items = array_merge($items, self::fetchItems($data['next'], $uid)); + } + + return $items; + } + /** * Checks if the given contact url does support ActivityPub * diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index 0b80e9786..f6c9311fc 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -71,7 +71,7 @@ class Transmitter */ public static function getFollowers($owner, $page = null) { - $condition = ['rel' => [Contact::FOLLOWER, Contact::FRIEND], 'network' => Protocol::NATIVE_SUPPORT, 'uid' => $owner['uid'], + $condition = ['rel' => [Contact::FOLLOWER, Contact::FRIEND], 'network' => Protocol::FEDERATED, 'uid' => $owner['uid'], 'self' => false, 'deleted' => false, 'hidden' => false, 'archive' => false, 'pending' => false]; $count = DBA::count('contact', $condition); @@ -120,7 +120,7 @@ class Transmitter */ public static function getFollowing($owner, $page = null) { - $condition = ['rel' => [Contact::SHARING, Contact::FRIEND], 'network' => Protocol::NATIVE_SUPPORT, 'uid' => $owner['uid'], + $condition = ['rel' => [Contact::SHARING, Contact::FRIEND], 'network' => Protocol::FEDERATED, 'uid' => $owner['uid'], 'self' => false, 'deleted' => false, 'hidden' => false, 'archive' => false, 'pending' => false]; $count = DBA::count('contact', $condition); @@ -341,7 +341,11 @@ class Transmitter } foreach ($activity[$element] as $receiver) { - if ($receiver == $profile['followers'] && !empty($item_profile['followers'])) { + if (empty($receiver)) { + continue; + } + + if (!empty($profile['followers']) && $receiver == $profile['followers'] && !empty($item_profile['followers'])) { $permissions[$element][] = $item_profile['followers']; } elseif (!in_array($receiver, $exclude)) { $permissions[$element][] = $receiver; @@ -649,7 +653,7 @@ class Transmitter $blindcopy = in_array($element, ['bto', 'bcc']); foreach ($permissions[$element] as $receiver) { - if (Network::isUrlBlocked($receiver)) { + if (empty($receiver) || Network::isUrlBlocked($receiver)) { continue; } diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php index 7bdaec464..fda1a650b 100644 --- a/src/Worker/UpdateGContact.php +++ b/src/Worker/UpdateGContact.php @@ -22,22 +22,28 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\DI; use Friendica\Model\GContact; -use Friendica\Database\DBA; class UpdateGContact { /** * Update global contact via probe - * @param string $url Global contact url - * @param string $command + * @param string $url Global contact url + * @param string $command + * @param integer $following_gcid gcontact ID of the contact that is followed by this one + * @param integer $follower_gcid gcontact ID of the contact that is following this one */ - public static function execute($url, $command = '') + public static function execute(string $url, string $command = '', int $following_gcid = 0, int $follower_gcid = 0) { $force = ($command == "force"); $success = GContact::updateFromProbe($url, $force); Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]); + + if ($success && DI::config()->get('system', 'gcontact_discovery')) { + GContact::discoverFollowers($url, $following_gcid, $follower_gcid); + } } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 0c0b28941..8cd01b4ae 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1335); + define('DB_UPDATE_VERSION', 1336); } return [ @@ -462,6 +462,7 @@ return [ "updated" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], + "last_discovery" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last contact discovery"], "archive_date" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "archived" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], "location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], @@ -490,6 +491,18 @@ return [ "updated" => ["updated"], ] ], + "gfollower" => [ + "comment" => "Followers of global contacts", + "fields" => [ + "gcid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["gcontact" => "id"], "comment" => "global contact"], + "follower-gcid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["gcontact" => "id"], "comment" => "global contact of the follower"], + "deleted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "1 indicates that the connection has been deleted"], + ], + "indexes" => [ + "PRIMARY" => ["gcid", "follower-gcid"], + "follower-gcid" => ["follower-gcid"], + ] + ], "glink" => [ "comment" => "'friends of friends' linkages derived from poco", "fields" => [ diff --git a/view/templates/admin/site.tpl b/view/templates/admin/site.tpl index 9ccda33ee..9e601b078 100644 --- a/view/templates/admin/site.tpl +++ b/view/templates/admin/site.tpl @@ -98,6 +98,7 @@

{{$portable_contacts}}

{{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_checkbox.tpl" field=$gcontact_discovery}} {{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery_since}} diff --git a/view/theme/frio/templates/admin/site.tpl b/view/theme/frio/templates/admin/site.tpl index e13b2960e..e4b05d453 100644 --- a/view/theme/frio/templates/admin/site.tpl +++ b/view/theme/frio/templates/admin/site.tpl @@ -213,6 +213,7 @@
{{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_checkbox.tpl" field=$gcontact_discovery}} {{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery_since}}