From d504789c3482277393dfbbc1a4e9906ecdbb228c Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 3 Oct 2020 10:52:34 +0000 Subject: [PATCH] Reduce the amount of contact updates --- src/Model/Contact.php | 28 ++++++++------ src/Module/Search/Index.php | 2 +- src/Worker/PullDirectory.php | 3 +- src/Worker/UpdatePublicContacts.php | 56 +++++++++++++++++++--------- src/Worker/UpdateServerDirectory.php | 4 +- 5 files changed, 60 insertions(+), 33 deletions(-) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 878f683b14..e997bffdef 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -225,7 +225,7 @@ class Contact // Add internal fields $removal = []; if (!empty($fields)) { - foreach (['id', 'avatar', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) { + foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) { if (!in_array($internal, $fields)) { $fields[] = $internal; $removal[] = $internal; @@ -255,9 +255,8 @@ class Contact } // Update the contact in the background if needed - $updated = max($contact['success_update'], $contact['updated'], $contact['last-update'], $contact['failure_update']); - if ((($updated < DateTimeFormat::utc('now -7 days')) || empty($contact['avatar'])) && - in_array($contact['network'], Protocol::FEDERATED)) { + $updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']); + if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED)) { Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']); } @@ -1859,10 +1858,6 @@ class Contact return false; } - if (Contact\Relation::isDiscoverable($ret['url'])) { - Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']); - } - if (isset($ret['hide']) && is_bool($ret['hide'])) { $ret['unsearchable'] = $ret['hide']; } @@ -1911,6 +1906,10 @@ class Contact if (!$update) { self::updateContact($id, $uid, $ret['url'], ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]); + if (Contact\Relation::isDiscoverable($ret['url'])) { + Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']); + } + // Update the public contact if ($uid != 0) { $contact = self::getByURL($ret['url'], false, ['id']); @@ -1948,6 +1947,10 @@ class Contact self::updateContact($id, $uid, $ret['url'], $ret); + if (Contact\Relation::isDiscoverable($ret['url'])) { + Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']); + } + return true; } @@ -2705,21 +2708,24 @@ class Contact { $added = 0; $updated = 0; + $unchanged = 0; $count = 0; foreach ($urls as $url) { - $contact = Contact::getByURL($url, false, ['id']); + $contact = Contact::getByURL($url, false, ['id', 'updated']); if (empty($contact['id'])) { Worker::add(PRIORITY_LOW, 'AddContact', 0, $url); ++$added; - } else { + } elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) { Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']); ++$updated; + } else { + ++$unchanged; } ++$count; } - return ['count' => $count, 'added' => $added, 'updated' => $updated]; + return ['count' => $count, 'added' => $added, 'updated' => $updated, 'unchanged' => $unchanged]; } /** diff --git a/src/Module/Search/Index.php b/src/Module/Search/Index.php index 82fe5543a8..5f776577e0 100644 --- a/src/Module/Search/Index.php +++ b/src/Module/Search/Index.php @@ -156,7 +156,7 @@ class Index extends BaseSearch } $last_uriid = isset($_GET['last_uriid']) ? intval($_GET['last_uriid']) : 0; -Logger::info('Blubb', ['uri' => $last_uriid]); + $pager = new Pager(DI::l10n(), DI::args()->getQueryString(), $itemsPerPage); if ($tag) { diff --git a/src/Worker/PullDirectory.php b/src/Worker/PullDirectory.php index 625431518d..8592105b24 100644 --- a/src/Worker/PullDirectory.php +++ b/src/Worker/PullDirectory.php @@ -22,7 +22,6 @@ namespace Friendica\Worker; use Friendica\Core\Logger; -use Friendica\Core\Worker; use Friendica\DI; use Friendica\Model\Contact; @@ -65,6 +64,6 @@ class PullDirectory $now = $contacts['now'] ?? 0; DI::config()->set('system', 'last-directory-sync', $now); - Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'directory' => $directory]); + Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'directory' => $directory]); } } diff --git a/src/Worker/UpdatePublicContacts.php b/src/Worker/UpdatePublicContacts.php index 519478e111..2e8602b054 100644 --- a/src/Worker/UpdatePublicContacts.php +++ b/src/Worker/UpdatePublicContacts.php @@ -36,28 +36,50 @@ class UpdatePublicContacts public static function execute() { $count = 0; - $last_updated = DateTimeFormat::utc('now - 1 week'); - $condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?", - Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated]; + $ids = []; + $base_condition = ['network' => Protocol::FEDERATED, 'uid' => 0, 'self' => false]; - if (DI::config()->get('system', 'update_active_contacts')) { - $condition = DBA::mergeConditions($condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR - `id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR - `id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`))"]); + if (!DI::config()->get('system', 'update_active_contacts')) { + // Add every contact (mostly failed ones) that hadn't been updated for six months + $condition = DBA::mergeConditions($base_condition, + ["`last-update` < ?", DateTimeFormat::utc('now - 6 month')]); + $ids = self::getContactsToUpdate($condition, $ids); + + // Add every non failed contact that hadn't been updated for a month + $condition = DBA::mergeConditions($base_condition, + ["NOT `failed` AND `last-update` < ?", DateTimeFormat::utc('now - 1 month')]); + $ids = self::getContactsToUpdate($condition, $ids); } - $oldest_date = ''; - $oldest_id = ''; - $contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]); - while ($contact = DBA::fetch($contacts)) { - if (empty($oldest_id)) { - $oldest_id = $contact['id']; - $oldest_date = $contact['last-update']; - } - Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']); + // Add every contact our system interacted with and hadn't been updated for a week + $condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR + `id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR + `id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`)) AND + `last-update` < ?", DateTimeFormat::utc('now - 1 week')]); + $ids = self::getContactsToUpdate($condition, $ids); + + foreach ($ids as $id) { + Worker::add(PRIORITY_LOW, "UpdateContact", $id); ++$count; } - Logger::info('Initiated update for public contacts', ['interval' => $count, 'id' => $oldest_id, 'oldest' => $oldest_date]); + + Logger::info('Initiated update for public contacts', ['count' => $count]); + } + + /** + * Returns contact ids based on a given condition + * + * @param array $condition + * @param array $ids + * @return array contact ids + */ + private static function getContactsToUpdate(array $condition, array $ids = []) + { + $contacts = DBA::select('contact', ['id'], $condition, ['limit' => 100, 'order' => ['last-update']]); + while ($contact = DBA::fetch($contacts)) { + $ids[] = $contact['id']; + } DBA::close($contacts); + return $ids; } } diff --git a/src/Worker/UpdateServerDirectory.php b/src/Worker/UpdateServerDirectory.php index 21599b533e..e6aebea8d8 100644 --- a/src/Worker/UpdateServerDirectory.php +++ b/src/Worker/UpdateServerDirectory.php @@ -73,7 +73,7 @@ class UpdateServerDirectory $result = Contact::addByUrls($urls); - Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'poco' => $gserver['poco']]); + Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'poco' => $gserver['poco']]); } private static function discoverMastodonDirectory(array $gserver) @@ -101,6 +101,6 @@ class UpdateServerDirectory $result = Contact::addByUrls($urls); - Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'url' => $gserver['url']]); + Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'url' => $gserver['url']]); } }