Merge pull request #8401 from annando/gfollower-epilogue

Reduce the load of "gfollower"

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 2020.03-dev (Dalmatian Bellflower)
--- DB_UPDATE_VERSION 1337
+-- Friendica 2020.03-rc (Dalmatian Bellflower)
+-- DB_UPDATE_VERSION 1338
 -- ------------------------------------------
 
 
@@ -254,6 +254,17 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	 INDEX `issued-id` (`issued-id`(64))
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table';
 
+--
+-- TABLE contact-relation
+--
+CREATE TABLE IF NOT EXISTS `contact-relation` (
+	`cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'contact the related contact had interacted with',
+	`relation-cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'related contact who had interacted with the contact',
+	`last-interaction` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last interaction',
+	 PRIMARY KEY(`cid`,`relation-cid`),
+	 INDEX `relation-cid` (`relation-cid`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Contact relations';
+
 --
 -- TABLE conv
 --

src/Model/Contact.php

@@ -1621,6 +1621,9 @@ class Contact
 				// Update the gcontact entry
 				if ($uid == 0) {
 					GContact::updateFromPublicContactID($contact_id);
+					if (($data['network'] == Protocol::ACTIVITYPUB) && in_array(DI::config()->get('system', 'gcontact_discovery'), [GContact::DISCOVERY_DIRECT, GContact::DISCOVERY_RECURSIVE])) {
+						GContact::discoverFollowers($data['url']);
+					}
 				}
 			}
 		} else {

src/Model/GContact.php

@@ -43,6 +43,19 @@ use Friendica\Util\Strings;
  */
 class GContact
 {
+	/**
+	 * No discovery of followers/followings
+	 */
+	const DISCOVERY_NONE = 0;
+	/**
+	 * Only discover followers/followings from direct contacts
+	 */
+	const DISCOVERY_DIRECT = 1;
+	/**
+	 * Recursive discovery of followers/followings
+	 */
+	const DISCOVERY_RECURSIVE = 2;
+
 	/**
 	 * Search global contact table by nick or name
 	 *

src/Model/Item.php

@@ -1695,7 +1695,7 @@ class Item
 
 			$fields = ['uri', 'parent-uri', 'id', 'deleted',
 				'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
-				'wall', 'private', 'forum_mode', 'origin'];
+				'wall', 'private', 'forum_mode', 'origin', 'author-id'];
 			$condition = ['uri' => $item['parent-uri'], 'uid' => $item['uid']];
 			$params = ['order' => ['id' => false]];
 			$parent = self::selectFirst($fields, $condition, $params);
@@ -1750,6 +1750,11 @@ class Item
 					DBA::update('thread', ['mention' => true], ['iid' => $parent_id]);
 					Logger::log('tagged thread ' . $parent_id . ' as mention for user ' . $item['uid'], Logger::DEBUG);
 				}
+
+				// Update the contact relations
+				if ($item['author-id'] != $parent['author-id']) {
+					DBA::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true);
+				}
 			} else {
 				/*
 				 * Allow one to see reply tweets from status.net even when

src/Module/Admin/Site.php

@@ -28,6 +28,7 @@ use Friendica\Core\Theme;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\GContact;
 use Friendica\Module\BaseAdmin;
 use Friendica\Module\Register;
 use Friendica\Protocol\PortableContact;
@@ -178,7 +179,7 @@ class Site extends BaseAdmin
 		$optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100);
 		$optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30);
 		$poco_completion        = (!empty($_POST['poco_completion'])        ? intval(trim($_POST['poco_completion']))        : false);
-		$gcontact_discovery     = (!empty($_POST['gcontact_discovery'])     ? intval(trim($_POST['gcontact_discovery']))     : false);
+		$gcontact_discovery     = (!empty($_POST['gcontact_discovery'])     ? intval(trim($_POST['gcontact_discovery']))     : GContact::DISCOVERY_NONE);
 		$poco_requery_days      = (!empty($_POST['poco_requery_days'])      ? intval(trim($_POST['poco_requery_days']))      : 7);
 		$poco_discovery         = (!empty($_POST['poco_discovery'])         ? intval(trim($_POST['poco_discovery']))         : PortableContact::DISABLED);
 		$poco_discovery_since   = (!empty($_POST['poco_discovery_since'])   ? intval(trim($_POST['poco_discovery_since']))   : 30);
@@ -549,6 +550,12 @@ class Site extends BaseAdmin
 			'develop' => DI::l10n()->t('check the development version')
 		];
 
+		$discovery_choices = [
+			GContact::DISCOVERY_NONE => DI::l10n()->t('none'),
+			GContact::DISCOVERY_DIRECT => DI::l10n()->t('Direct contacts'),
+			GContact::DISCOVERY_RECURSIVE => DI::l10n()->t('Contacts of contacts')
+		];
+
 		$diaspora_able = (DI::baseUrl()->getUrlPath() == '');
 
 		$optimize_max_tablesize = DI::config()->get('system', 'optimize_max_tablesize', -1);
@@ -620,7 +627,7 @@ class Site extends BaseAdmin
 			'$language'         => ['language', DI::l10n()->t('System language'), DI::config()->get('system', 'language'), '', $lang_choices],
 			'$theme'            => ['theme', DI::l10n()->t('System theme'), DI::config()->get('system', 'theme'), DI::l10n()->t('Default system theme - may be over-ridden by user profiles - <a href="/admin/themes" id="cnftheme">Change default theme settings</a>'), $theme_choices],
 			'$theme_mobile'     => ['theme_mobile', DI::l10n()->t('Mobile system theme'), DI::config()->get('system', 'mobile-theme', '---'), DI::l10n()->t('Theme for mobile devices'), $theme_choices_mobile],
-			'$ssl_policy'       => ['ssl_policy', DI::l10n()->t('SSL link policy'), (string)intval(DI::config()->get('system', 'ssl_policy')), DI::l10n()->t('Determines whether generated links should be forced to use SSL'), $ssl_choices],
+			'$ssl_policy'       => ['ssl_policy', DI::l10n()->t('SSL link policy'), DI::config()->get('system', 'ssl_policy'), DI::l10n()->t('Determines whether generated links should be forced to use SSL'), $ssl_choices],
 			'$force_ssl'        => ['force_ssl', DI::l10n()->t('Force SSL'), DI::config()->get('system', 'force_ssl'), DI::l10n()->t('Force all Non-SSL requests to SSL - Attention: on some systems it could lead to endless loops.')],
 			'$hide_help'        => ['hide_help', DI::l10n()->t('Hide help entry from navigation menu'), DI::config()->get('system', 'hide_help'), DI::l10n()->t('Hides the menu entry for the Help pages from the navigation menu. You can still access it calling /help directly.')],
 			'$singleuser'       => ['singleuser', DI::l10n()->t('Single user instance'), DI::config()->get('system', 'singleuser', '---'), DI::l10n()->t('Make this instance multi-user or single-user for the named user'), $user_names],
@@ -671,10 +678,10 @@ class Site extends BaseAdmin
 			'$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')],
 
 			'$poco_completion'        => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')],
-			'$gcontact_discovery'     => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.')],
+			'$gcontact_discovery'     => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.'), $discovery_choices],
 			'$poco_requery_days'      => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')],
-			'$poco_discovery'         => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices],
-			'$poco_discovery_since'   => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices],
+			'$poco_discovery'         => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), DI::config()->get('system', 'poco_discovery'), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices],
+			'$poco_discovery_since'   => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), DI::config()->get('system', 'poco_discovery_since'), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices],
 			'$poco_local_search'      => ['poco_local_search', DI::l10n()->t('Search the local directory'), DI::config()->get('system', 'poco_local_search'), DI::l10n()->t('Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.')],
 
 			'$nodeinfo'               => ['nodeinfo', DI::l10n()->t('Publish server information'), DI::config()->get('system', 'nodeinfo'), DI::l10n()->t('If enabled, general server and usage data will be published. The data contains the name and version of the server, number of users with public profiles, number of posts and the activated protocols and connectors. See <a href="http://the-federation.info/">the-federation.info</a> for details.')],

src/Worker/UpdateContact.php

@@ -23,8 +23,6 @@ namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
 use Friendica\Model\Contact;
-use Friendica\Util\DateTimeFormat;
-use Friendica\Database\DBA;
 
 class UpdateContact
 {

src/Worker/UpdateGContact.php

@@ -40,7 +40,7 @@ class UpdateGContact
 
 		Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]);
 
-		if ($success && DI::config()->get('system', 'gcontact_discovery')) {
+		if ($success && (DI::config()->get('system', 'gcontact_discovery') == GContact::DISCOVERY_RECURSIVE)) {
 			GContact::discoverFollowers($url);
 		}
 	}

static/dbstructure.config.php

@@ -51,7 +51,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1337);
+	define('DB_UPDATE_VERSION', 1338);
 }
 
 return [
@@ -317,6 +317,18 @@ return [
 			"issued-id" => ["issued-id(64)"],
 		]
 	],
+	"contact-relation" => [
+		"comment" => "Contact relations",
+		"fields" => [
+			"cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact the related contact had interacted with"],
+			"relation-cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "related contact who had interacted with the contact"],
+			"last-interaction" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last interaction"],
+		],
+		"indexes" => [
+			"PRIMARY" => ["cid", "relation-cid"],
+			"relation-cid" => ["relation-cid"],
+		]
+	],
 	"conv" => [
 		"comment" => "private messages",
 		"fields" => [

view/templates/admin/site.tpl

@@ -98,7 +98,7 @@
 
 	<h3>{{$portable_contacts}}</h3>
 	{{include file="field_checkbox.tpl" field=$poco_completion}}
-	{{include file="field_checkbox.tpl" field=$gcontact_discovery}}
+	{{include file="field_select.tpl" field=$gcontact_discovery}}
 	{{include file="field_input.tpl" field=$poco_requery_days}}
 	{{include file="field_select.tpl" field=$poco_discovery}}
 	{{include file="field_select.tpl" field=$poco_discovery_since}}

view/theme/frio/templates/admin/site.tpl

@@ -213,7 +213,7 @@
 				<div id="admin-settings-contacts-collapse" class="panel-collapse collapse" role="tabpanel" aria-labelledby="admin-settings-cocontactsrporate">
 
 					{{include file="field_checkbox.tpl" field=$poco_completion}}
-					{{include file="field_checkbox.tpl" field=$gcontact_discovery}}
+					{{include file="field_select.tpl" field=$gcontact_discovery}}
 					{{include file="field_input.tpl" field=$poco_requery_days}}
 					{{include file="field_select.tpl" field=$poco_discovery}}
 					{{include file="field_select.tpl" field=$poco_discovery_since}}