Reduce the amount of contact updates

This commit is contained in:
Michael 2020-10-03 10:52:34 +00:00
parent 8623c075af
commit d504789c34
5 changed files with 60 additions and 33 deletions

View file

@ -225,7 +225,7 @@ class Contact
// Add internal fields // Add internal fields
$removal = []; $removal = [];
if (!empty($fields)) { if (!empty($fields)) {
foreach (['id', 'avatar', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) { foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) {
if (!in_array($internal, $fields)) { if (!in_array($internal, $fields)) {
$fields[] = $internal; $fields[] = $internal;
$removal[] = $internal; $removal[] = $internal;
@ -255,9 +255,8 @@ class Contact
} }
// Update the contact in the background if needed // Update the contact in the background if needed
$updated = max($contact['success_update'], $contact['updated'], $contact['last-update'], $contact['failure_update']); $updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']);
if ((($updated < DateTimeFormat::utc('now -7 days')) || empty($contact['avatar'])) && if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED)) {
in_array($contact['network'], Protocol::FEDERATED)) {
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']); Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
} }
@ -1859,10 +1858,6 @@ class Contact
return false; return false;
} }
if (Contact\Relation::isDiscoverable($ret['url'])) {
Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']);
}
if (isset($ret['hide']) && is_bool($ret['hide'])) { if (isset($ret['hide']) && is_bool($ret['hide'])) {
$ret['unsearchable'] = $ret['hide']; $ret['unsearchable'] = $ret['hide'];
} }
@ -1911,6 +1906,10 @@ class Contact
if (!$update) { if (!$update) {
self::updateContact($id, $uid, $ret['url'], ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]); self::updateContact($id, $uid, $ret['url'], ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]);
if (Contact\Relation::isDiscoverable($ret['url'])) {
Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']);
}
// Update the public contact // Update the public contact
if ($uid != 0) { if ($uid != 0) {
$contact = self::getByURL($ret['url'], false, ['id']); $contact = self::getByURL($ret['url'], false, ['id']);
@ -1948,6 +1947,10 @@ class Contact
self::updateContact($id, $uid, $ret['url'], $ret); self::updateContact($id, $uid, $ret['url'], $ret);
if (Contact\Relation::isDiscoverable($ret['url'])) {
Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']);
}
return true; return true;
} }
@ -2705,21 +2708,24 @@ class Contact
{ {
$added = 0; $added = 0;
$updated = 0; $updated = 0;
$unchanged = 0;
$count = 0; $count = 0;
foreach ($urls as $url) { foreach ($urls as $url) {
$contact = Contact::getByURL($url, false, ['id']); $contact = Contact::getByURL($url, false, ['id', 'updated']);
if (empty($contact['id'])) { if (empty($contact['id'])) {
Worker::add(PRIORITY_LOW, 'AddContact', 0, $url); Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
++$added; ++$added;
} else { } elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) {
Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']); Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']);
++$updated; ++$updated;
} else {
++$unchanged;
} }
++$count; ++$count;
} }
return ['count' => $count, 'added' => $added, 'updated' => $updated]; return ['count' => $count, 'added' => $added, 'updated' => $updated, 'unchanged' => $unchanged];
} }
/** /**

View file

@ -156,7 +156,7 @@ class Index extends BaseSearch
} }
$last_uriid = isset($_GET['last_uriid']) ? intval($_GET['last_uriid']) : 0; $last_uriid = isset($_GET['last_uriid']) ? intval($_GET['last_uriid']) : 0;
Logger::info('Blubb', ['uri' => $last_uriid]);
$pager = new Pager(DI::l10n(), DI::args()->getQueryString(), $itemsPerPage); $pager = new Pager(DI::l10n(), DI::args()->getQueryString(), $itemsPerPage);
if ($tag) { if ($tag) {

View file

@ -22,7 +22,6 @@
namespace Friendica\Worker; namespace Friendica\Worker;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\DI; use Friendica\DI;
use Friendica\Model\Contact; use Friendica\Model\Contact;
@ -65,6 +64,6 @@ class PullDirectory
$now = $contacts['now'] ?? 0; $now = $contacts['now'] ?? 0;
DI::config()->set('system', 'last-directory-sync', $now); DI::config()->set('system', 'last-directory-sync', $now);
Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'directory' => $directory]); Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'directory' => $directory]);
} }
} }

View file

@ -36,28 +36,50 @@ class UpdatePublicContacts
public static function execute() public static function execute()
{ {
$count = 0; $count = 0;
$last_updated = DateTimeFormat::utc('now - 1 week'); $ids = [];
$condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?", $base_condition = ['network' => Protocol::FEDERATED, 'uid' => 0, 'self' => false];
Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated];
if (DI::config()->get('system', 'update_active_contacts')) { if (!DI::config()->get('system', 'update_active_contacts')) {
$condition = DBA::mergeConditions($condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR // Add every contact (mostly failed ones) that hadn't been updated for six months
`id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR $condition = DBA::mergeConditions($base_condition,
`id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`))"]); ["`last-update` < ?", DateTimeFormat::utc('now - 6 month')]);
$ids = self::getContactsToUpdate($condition, $ids);
// Add every non failed contact that hadn't been updated for a month
$condition = DBA::mergeConditions($base_condition,
["NOT `failed` AND `last-update` < ?", DateTimeFormat::utc('now - 1 month')]);
$ids = self::getContactsToUpdate($condition, $ids);
} }
$oldest_date = ''; // Add every contact our system interacted with and hadn't been updated for a week
$oldest_id = ''; $condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR
$contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]); `id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR
while ($contact = DBA::fetch($contacts)) { `id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`)) AND
if (empty($oldest_id)) { `last-update` < ?", DateTimeFormat::utc('now - 1 week')]);
$oldest_id = $contact['id']; $ids = self::getContactsToUpdate($condition, $ids);
$oldest_date = $contact['last-update'];
} foreach ($ids as $id) {
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']); Worker::add(PRIORITY_LOW, "UpdateContact", $id);
++$count; ++$count;
} }
Logger::info('Initiated update for public contacts', ['interval' => $count, 'id' => $oldest_id, 'oldest' => $oldest_date]);
Logger::info('Initiated update for public contacts', ['count' => $count]);
}
/**
* Returns contact ids based on a given condition
*
* @param array $condition
* @param array $ids
* @return array contact ids
*/
private static function getContactsToUpdate(array $condition, array $ids = [])
{
$contacts = DBA::select('contact', ['id'], $condition, ['limit' => 100, 'order' => ['last-update']]);
while ($contact = DBA::fetch($contacts)) {
$ids[] = $contact['id'];
}
DBA::close($contacts); DBA::close($contacts);
return $ids;
} }
} }

View file

@ -73,7 +73,7 @@ class UpdateServerDirectory
$result = Contact::addByUrls($urls); $result = Contact::addByUrls($urls);
Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'poco' => $gserver['poco']]); Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'poco' => $gserver['poco']]);
} }
private static function discoverMastodonDirectory(array $gserver) private static function discoverMastodonDirectory(array $gserver)
@ -101,6 +101,6 @@ class UpdateServerDirectory
$result = Contact::addByUrls($urls); $result = Contact::addByUrls($urls);
Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'url' => $gserver['url']]); Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'url' => $gserver['url']]);
} }
} }