From f27900ad7808036402716ad3299c9fbdf4b94b3c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 10 Mar 2020 21:10:59 +0000 Subject: [PATCH 1/4] Reduce the load of "gfollower" --- src/Model/Contact.php | 3 +++ src/Model/GContact.php | 13 +++++++++++++ src/Model/Item.php | 12 ++++++++++-- src/Module/Admin/Site.php | 11 +++++++++-- src/Worker/UpdateContact.php | 2 -- src/Worker/UpdateGContact.php | 2 +- static/dbstructure.config.php | 13 ++++++++++++- view/templates/admin/site.tpl | 2 +- view/theme/frio/templates/admin/site.tpl | 2 +- 9 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index f199012f1..9ae27994c 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1621,6 +1621,9 @@ class Contact // Update the gcontact entry if ($uid == 0) { GContact::updateFromPublicContactID($contact_id); + if (($data['network'] == Protocol::ACTIVITYPUB) && in_array(DI::config()->get('system', 'gcontact_discovery'), [GContact::DISCOVERY_DIRECT, GContact::DISCOVERY_RECURSIVE])) { + GContact::discoverFollowers($data['url']); + } } } } else { diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 19ab87ca4..59c84bc49 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -43,6 +43,19 @@ use Friendica\Util\Strings; */ class GContact { + /** + * No discovery of followers/followings + */ + const DISCOVERY_NONE = 0; + /** + * Only discover followers/followings from direct contacts + */ + const DISCOVERY_DIRECT = 1; + /** + * Recursive discovery of followers/followings + */ + const DISCOVERY_RECURSIVE = 2; + /** * Search global contact table by nick or name * diff --git a/src/Model/Item.php b/src/Model/Item.php index fe62e888d..3acf95988 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1695,7 +1695,7 @@ class Item $fields = ['uri', 'parent-uri', 'id', 'deleted', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', - 'wall', 'private', 'forum_mode', 'origin']; + 'wall', 'private', 'forum_mode', 'origin', 'author-id']; $condition = ['uri' => $item['parent-uri'], 'uid' => $item['uid']]; $params = ['order' => ['id' => false]]; $parent = self::selectFirst($fields, $condition, $params); @@ -1750,7 +1750,15 @@ class Item DBA::update('thread', ['mention' => true], ['iid' => $parent_id]); Logger::log('tagged thread ' . $parent_id . ' as mention for user ' . $item['uid'], Logger::DEBUG); } - } else { + + // Update the contact relations + if ($item['author-id'] != $parent['author-id']) { + $fields = ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']]; + if (!DBA::exists('contact-relation', $fields)) { + DBA::insert('contact-relation', $fields, true); + } + } + } else { /* * Allow one to see reply tweets from status.net even when * we don't have or can't see the original post. diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index 846280c4f..00ca20347 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -28,6 +28,7 @@ use Friendica\Core\Theme; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\GContact; use Friendica\Module\BaseAdmin; use Friendica\Module\Register; use Friendica\Protocol\PortableContact; @@ -178,7 +179,7 @@ class Site extends BaseAdmin $optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100); $optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30); $poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false); - $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : false); + $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : 0); $poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7); $poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED); $poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30); @@ -549,6 +550,12 @@ class Site extends BaseAdmin 'develop' => DI::l10n()->t('check the development version') ]; + $discovery_choices = [ + GContact::DISCOVERY_NONE => DI::l10n()->t('none'), + GContact::DISCOVERY_DIRECT => DI::l10n()->t('Direct contacts'), + GContact::DISCOVERY_RECURSIVE => DI::l10n()->t('Contacts of contacts') + ]; + $diaspora_able = (DI::baseUrl()->getUrlPath() == ''); $optimize_max_tablesize = DI::config()->get('system', 'optimize_max_tablesize', -1); @@ -671,7 +678,7 @@ class Site extends BaseAdmin '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], '$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')], - '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.')], + '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), (string)intval(DI::config()->get('system', 'gcontact_discovery')), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.'), $discovery_choices], '$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')], '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], diff --git a/src/Worker/UpdateContact.php b/src/Worker/UpdateContact.php index 342e99550..67bc45ef9 100644 --- a/src/Worker/UpdateContact.php +++ b/src/Worker/UpdateContact.php @@ -23,8 +23,6 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Model\Contact; -use Friendica\Util\DateTimeFormat; -use Friendica\Database\DBA; class UpdateContact { diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php index b88e0899e..e06356ad4 100644 --- a/src/Worker/UpdateGContact.php +++ b/src/Worker/UpdateGContact.php @@ -40,7 +40,7 @@ class UpdateGContact Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]); - if ($success && DI::config()->get('system', 'gcontact_discovery')) { + if ($success && (DI::config()->get('system', 'gcontact_discovery') == GContact::DISCOVERY_RECURSIVE)) { GContact::discoverFollowers($url); } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index cfbee73fb..94a6e6ec2 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1337); + define('DB_UPDATE_VERSION', 1338); } return [ @@ -317,6 +317,17 @@ return [ "issued-id" => ["issued-id(64)"], ] ], + "contact-relation" => [ + "comment" => "Contact relations", + "fields" => [ + "cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact the related contact had interacted with"], + "relation-cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "related contact who had interacted with the contact"], + ], + "indexes" => [ + "PRIMARY" => ["cid", "relation-cid"], + "relation-cid" => ["relation-cid"], + ] + ], "conv" => [ "comment" => "private messages", "fields" => [ diff --git a/view/templates/admin/site.tpl b/view/templates/admin/site.tpl index 9e601b078..b6d0c5930 100644 --- a/view/templates/admin/site.tpl +++ b/view/templates/admin/site.tpl @@ -98,7 +98,7 @@

{{$portable_contacts}}

{{include file="field_checkbox.tpl" field=$poco_completion}} - {{include file="field_checkbox.tpl" field=$gcontact_discovery}} + {{include file="field_select.tpl" field=$gcontact_discovery}} {{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery_since}} diff --git a/view/theme/frio/templates/admin/site.tpl b/view/theme/frio/templates/admin/site.tpl index e4b05d453..902b8a000 100644 --- a/view/theme/frio/templates/admin/site.tpl +++ b/view/theme/frio/templates/admin/site.tpl @@ -213,7 +213,7 @@
{{include file="field_checkbox.tpl" field=$poco_completion}} - {{include file="field_checkbox.tpl" field=$gcontact_discovery}} + {{include file="field_select.tpl" field=$gcontact_discovery}} {{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery_since}} From 07a487cfdc9906478691e76d413e8efa191f478c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 10 Mar 2020 21:21:48 +0000 Subject: [PATCH 2/4] Fixed indentation --- src/Model/Item.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index 3acf95988..319abf6b2 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1758,7 +1758,7 @@ class Item DBA::insert('contact-relation', $fields, true); } } - } else { + } else { /* * Allow one to see reply tweets from status.net even when * we don't have or can't see the original post. From c261457b035e598c4b0b14c74bbb437d69926dfa Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 10 Mar 2020 22:04:03 +0000 Subject: [PATCH 3/4] Store the interaction date as well. --- database.sql | 15 +++++++++++++-- src/Model/Item.php | 5 +---- static/dbstructure.config.php | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/database.sql b/database.sql index f33f89211..4956798ba 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 2020.03-dev (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1337 +-- Friendica 2020.03-rc (Dalmatian Bellflower) +-- DB_UPDATE_VERSION 1338 -- ------------------------------------------ @@ -254,6 +254,17 @@ CREATE TABLE IF NOT EXISTS `contact` ( INDEX `issued-id` (`issued-id`(64)) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table'; +-- +-- TABLE contact-relation +-- +CREATE TABLE IF NOT EXISTS `contact-relation` ( + `cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'contact the related contact had interacted with', + `relation-cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'related contact who had interacted with the contact', + `last-interaction` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last interaction', + PRIMARY KEY(`cid`,`relation-cid`), + INDEX `relation-cid` (`relation-cid`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Contact relations'; + -- -- TABLE conv -- diff --git a/src/Model/Item.php b/src/Model/Item.php index 319abf6b2..b9aeff4c9 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1753,10 +1753,7 @@ class Item // Update the contact relations if ($item['author-id'] != $parent['author-id']) { - $fields = ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']]; - if (!DBA::exists('contact-relation', $fields)) { - DBA::insert('contact-relation', $fields, true); - } + dba::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true); } } else { /* diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 94a6e6ec2..b8daa4eee 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -322,6 +322,7 @@ return [ "fields" => [ "cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact the related contact had interacted with"], "relation-cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "related contact who had interacted with the contact"], + "last-interaction" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last interaction"], ], "indexes" => [ "PRIMARY" => ["cid", "relation-cid"], From 59f58f999d5cc5699efa21f508d906284f124c27 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 10 Mar 2020 22:51:05 +0000 Subject: [PATCH 4/4] Small fixes --- src/Model/Item.php | 2 +- src/Module/Admin/Site.php | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index b9aeff4c9..1205516fa 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1753,7 +1753,7 @@ class Item // Update the contact relations if ($item['author-id'] != $parent['author-id']) { - dba::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true); + DBA::update('contact-relation', ['last-interaction' => $item['created']], ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']], true); } } else { /* diff --git a/src/Module/Admin/Site.php b/src/Module/Admin/Site.php index 00ca20347..501edaf4d 100644 --- a/src/Module/Admin/Site.php +++ b/src/Module/Admin/Site.php @@ -179,7 +179,7 @@ class Site extends BaseAdmin $optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100); $optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30); $poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false); - $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : 0); + $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : GContact::DISCOVERY_NONE); $poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7); $poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED); $poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30); @@ -627,7 +627,7 @@ class Site extends BaseAdmin '$language' => ['language', DI::l10n()->t('System language'), DI::config()->get('system', 'language'), '', $lang_choices], '$theme' => ['theme', DI::l10n()->t('System theme'), DI::config()->get('system', 'theme'), DI::l10n()->t('Default system theme - may be over-ridden by user profiles - Change default theme settings'), $theme_choices], '$theme_mobile' => ['theme_mobile', DI::l10n()->t('Mobile system theme'), DI::config()->get('system', 'mobile-theme', '---'), DI::l10n()->t('Theme for mobile devices'), $theme_choices_mobile], - '$ssl_policy' => ['ssl_policy', DI::l10n()->t('SSL link policy'), (string)intval(DI::config()->get('system', 'ssl_policy')), DI::l10n()->t('Determines whether generated links should be forced to use SSL'), $ssl_choices], + '$ssl_policy' => ['ssl_policy', DI::l10n()->t('SSL link policy'), DI::config()->get('system', 'ssl_policy'), DI::l10n()->t('Determines whether generated links should be forced to use SSL'), $ssl_choices], '$force_ssl' => ['force_ssl', DI::l10n()->t('Force SSL'), DI::config()->get('system', 'force_ssl'), DI::l10n()->t('Force all Non-SSL requests to SSL - Attention: on some systems it could lead to endless loops.')], '$hide_help' => ['hide_help', DI::l10n()->t('Hide help entry from navigation menu'), DI::config()->get('system', 'hide_help'), DI::l10n()->t('Hides the menu entry for the Help pages from the navigation menu. You can still access it calling /help directly.')], '$singleuser' => ['singleuser', DI::l10n()->t('Single user instance'), DI::config()->get('system', 'singleuser', '---'), DI::l10n()->t('Make this instance multi-user or single-user for the named user'), $user_names], @@ -678,10 +678,10 @@ class Site extends BaseAdmin '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], '$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')], - '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), (string)intval(DI::config()->get('system', 'gcontact_discovery')), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.'), $discovery_choices], + '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.'), $discovery_choices], '$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')], - '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], - '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], + '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), DI::config()->get('system', 'poco_discovery'), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], + '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), DI::config()->get('system', 'poco_discovery_since'), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], '$poco_local_search' => ['poco_local_search', DI::l10n()->t('Search the local directory'), DI::config()->get('system', 'poco_local_search'), DI::l10n()->t('Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.')], '$nodeinfo' => ['nodeinfo', DI::l10n()->t('Publish server information'), DI::config()->get('system', 'nodeinfo'), DI::l10n()->t('If enabled, general server and usage data will be published. The data contains the name and version of the server, number of users with public profiles, number of posts and the activated protocols and connectors. See the-federation.info for details.')],