Reduce the load of "gfollower"

This commit is contained in:
Michael 2020-03-10 21:10:59 +00:00
parent 2b5ef80912
commit f27900ad78
9 changed files with 50 additions and 10 deletions

View file

@ -1621,6 +1621,9 @@ class Contact
// Update the gcontact entry // Update the gcontact entry
if ($uid == 0) { if ($uid == 0) {
GContact::updateFromPublicContactID($contact_id); GContact::updateFromPublicContactID($contact_id);
if (($data['network'] == Protocol::ACTIVITYPUB) && in_array(DI::config()->get('system', 'gcontact_discovery'), [GContact::DISCOVERY_DIRECT, GContact::DISCOVERY_RECURSIVE])) {
GContact::discoverFollowers($data['url']);
}
} }
} }
} else { } else {

View file

@ -43,6 +43,19 @@ use Friendica\Util\Strings;
*/ */
class GContact class GContact
{ {
/**
* No discovery of followers/followings
*/
const DISCOVERY_NONE = 0;
/**
* Only discover followers/followings from direct contacts
*/
const DISCOVERY_DIRECT = 1;
/**
* Recursive discovery of followers/followings
*/
const DISCOVERY_RECURSIVE = 2;
/** /**
* Search global contact table by nick or name * Search global contact table by nick or name
* *

View file

@ -1695,7 +1695,7 @@ class Item
$fields = ['uri', 'parent-uri', 'id', 'deleted', $fields = ['uri', 'parent-uri', 'id', 'deleted',
'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid',
'wall', 'private', 'forum_mode', 'origin']; 'wall', 'private', 'forum_mode', 'origin', 'author-id'];
$condition = ['uri' => $item['parent-uri'], 'uid' => $item['uid']]; $condition = ['uri' => $item['parent-uri'], 'uid' => $item['uid']];
$params = ['order' => ['id' => false]]; $params = ['order' => ['id' => false]];
$parent = self::selectFirst($fields, $condition, $params); $parent = self::selectFirst($fields, $condition, $params);
@ -1750,7 +1750,15 @@ class Item
DBA::update('thread', ['mention' => true], ['iid' => $parent_id]); DBA::update('thread', ['mention' => true], ['iid' => $parent_id]);
Logger::log('tagged thread ' . $parent_id . ' as mention for user ' . $item['uid'], Logger::DEBUG); Logger::log('tagged thread ' . $parent_id . ' as mention for user ' . $item['uid'], Logger::DEBUG);
} }
} else {
// Update the contact relations
if ($item['author-id'] != $parent['author-id']) {
$fields = ['cid' => $parent['author-id'], 'relation-cid' => $item['author-id']];
if (!DBA::exists('contact-relation', $fields)) {
DBA::insert('contact-relation', $fields, true);
}
}
} else {
/* /*
* Allow one to see reply tweets from status.net even when * Allow one to see reply tweets from status.net even when
* we don't have or can't see the original post. * we don't have or can't see the original post.

View file

@ -28,6 +28,7 @@ use Friendica\Core\Theme;
use Friendica\Core\Worker; use Friendica\Core\Worker;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Model\GContact;
use Friendica\Module\BaseAdmin; use Friendica\Module\BaseAdmin;
use Friendica\Module\Register; use Friendica\Module\Register;
use Friendica\Protocol\PortableContact; use Friendica\Protocol\PortableContact;
@ -178,7 +179,7 @@ class Site extends BaseAdmin
$optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100); $optimize_max_tablesize = (!empty($_POST['optimize_max_tablesize']) ? intval(trim($_POST['optimize_max_tablesize'])) : 100);
$optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30); $optimize_fragmentation = (!empty($_POST['optimize_fragmentation']) ? intval(trim($_POST['optimize_fragmentation'])) : 30);
$poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false); $poco_completion = (!empty($_POST['poco_completion']) ? intval(trim($_POST['poco_completion'])) : false);
$gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : false); $gcontact_discovery = (!empty($_POST['gcontact_discovery']) ? intval(trim($_POST['gcontact_discovery'])) : 0);
$poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7); $poco_requery_days = (!empty($_POST['poco_requery_days']) ? intval(trim($_POST['poco_requery_days'])) : 7);
$poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED); $poco_discovery = (!empty($_POST['poco_discovery']) ? intval(trim($_POST['poco_discovery'])) : PortableContact::DISABLED);
$poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30); $poco_discovery_since = (!empty($_POST['poco_discovery_since']) ? intval(trim($_POST['poco_discovery_since'])) : 30);
@ -549,6 +550,12 @@ class Site extends BaseAdmin
'develop' => DI::l10n()->t('check the development version') 'develop' => DI::l10n()->t('check the development version')
]; ];
$discovery_choices = [
GContact::DISCOVERY_NONE => DI::l10n()->t('none'),
GContact::DISCOVERY_DIRECT => DI::l10n()->t('Direct contacts'),
GContact::DISCOVERY_RECURSIVE => DI::l10n()->t('Contacts of contacts')
];
$diaspora_able = (DI::baseUrl()->getUrlPath() == ''); $diaspora_able = (DI::baseUrl()->getUrlPath() == '');
$optimize_max_tablesize = DI::config()->get('system', 'optimize_max_tablesize', -1); $optimize_max_tablesize = DI::config()->get('system', 'optimize_max_tablesize', -1);
@ -671,7 +678,7 @@ class Site extends BaseAdmin
'$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')], '$optimize_fragmentation' => ['optimize_fragmentation', DI::l10n()->t('Minimum level of fragmentation'), DI::config()->get('system', 'optimize_fragmentation', 30), DI::l10n()->t('Minimum fragmenation level to start the automatic optimization - default value is 30%.')],
'$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')], '$poco_completion' => ['poco_completion', DI::l10n()->t('Periodical check of global contacts'), DI::config()->get('system', 'poco_completion'), DI::l10n()->t('If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.')],
'$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), DI::config()->get('system', 'gcontact_discovery'), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.')], '$gcontact_discovery' => ['gcontact_discovery', DI::l10n()->t('Discover followers/followings from global contacts'), (string)intval(DI::config()->get('system', 'gcontact_discovery')), DI::l10n()->t('If enabled, the global contacts are checked for new contacts among their followers and following contacts. This option will create huge masses of jobs, so it should only be activated on powerful machines.'), $discovery_choices],
'$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')], '$poco_requery_days' => ['poco_requery_days', DI::l10n()->t('Days between requery'), DI::config()->get('system', 'poco_requery_days'), DI::l10n()->t('Number of days after which a server is requeried for his contacts.')],
'$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices], '$poco_discovery' => ['poco_discovery', DI::l10n()->t('Discover contacts from other servers'), (string)intval(DI::config()->get('system', 'poco_discovery')), DI::l10n()->t('Periodically query other servers for contacts. You can choose between "Users": the users on the remote system, "Global Contacts": active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren\'t available. The fallback increases the server load, so the recommended setting is "Users, Global Contacts".'), $poco_discovery_choices],
'$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices], '$poco_discovery_since' => ['poco_discovery_since', DI::l10n()->t('Timeframe for fetching global contacts'), (string)intval(DI::config()->get('system', 'poco_discovery_since')), DI::l10n()->t('When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers.'), $poco_discovery_since_choices],

View file

@ -23,8 +23,6 @@ namespace Friendica\Worker;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Model\Contact; use Friendica\Model\Contact;
use Friendica\Util\DateTimeFormat;
use Friendica\Database\DBA;
class UpdateContact class UpdateContact
{ {

View file

@ -40,7 +40,7 @@ class UpdateGContact
Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]); Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]);
if ($success && DI::config()->get('system', 'gcontact_discovery')) { if ($success && (DI::config()->get('system', 'gcontact_discovery') == GContact::DISCOVERY_RECURSIVE)) {
GContact::discoverFollowers($url); GContact::discoverFollowers($url);
} }
} }

View file

@ -51,7 +51,7 @@
use Friendica\Database\DBA; use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) { if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1337); define('DB_UPDATE_VERSION', 1338);
} }
return [ return [
@ -317,6 +317,17 @@ return [
"issued-id" => ["issued-id(64)"], "issued-id" => ["issued-id(64)"],
] ]
], ],
"contact-relation" => [
"comment" => "Contact relations",
"fields" => [
"cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact the related contact had interacted with"],
"relation-cid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "related contact who had interacted with the contact"],
],
"indexes" => [
"PRIMARY" => ["cid", "relation-cid"],
"relation-cid" => ["relation-cid"],
]
],
"conv" => [ "conv" => [
"comment" => "private messages", "comment" => "private messages",
"fields" => [ "fields" => [

View file

@ -98,7 +98,7 @@
<h3>{{$portable_contacts}}</h3> <h3>{{$portable_contacts}}</h3>
{{include file="field_checkbox.tpl" field=$poco_completion}} {{include file="field_checkbox.tpl" field=$poco_completion}}
{{include file="field_checkbox.tpl" field=$gcontact_discovery}} {{include file="field_select.tpl" field=$gcontact_discovery}}
{{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_input.tpl" field=$poco_requery_days}}
{{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery}}
{{include file="field_select.tpl" field=$poco_discovery_since}} {{include file="field_select.tpl" field=$poco_discovery_since}}

View file

@ -213,7 +213,7 @@
<div id="admin-settings-contacts-collapse" class="panel-collapse collapse" role="tabpanel" aria-labelledby="admin-settings-cocontactsrporate"> <div id="admin-settings-contacts-collapse" class="panel-collapse collapse" role="tabpanel" aria-labelledby="admin-settings-cocontactsrporate">
{{include file="field_checkbox.tpl" field=$poco_completion}} {{include file="field_checkbox.tpl" field=$poco_completion}}
{{include file="field_checkbox.tpl" field=$gcontact_discovery}} {{include file="field_select.tpl" field=$gcontact_discovery}}
{{include file="field_input.tpl" field=$poco_requery_days}} {{include file="field_input.tpl" field=$poco_requery_days}}
{{include file="field_select.tpl" field=$poco_discovery}} {{include file="field_select.tpl" field=$poco_discovery}}
{{include file="field_select.tpl" field=$poco_discovery_since}} {{include file="field_select.tpl" field=$poco_discovery_since}}