tobias/friendica
1
0
Fork 0
Code Pull requests Activity

Merge pull request #9346 from annando/reduce-contact-update

Browse source 
Reduce the amount of contact updates
This commit is contained in:
Hypolite Petovan 2020-10-03 12:43:28 -04:00 committed by GitHub
parent cfebf2115f d504789c34
commit d4968b4562
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 60 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/Model/Contact.php
View file

@ -225,7 +225,7 @@ class Contact
// Add internal fields
$removal = [];
if (!empty($fields)) {
foreach (['id', 'avatar', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) {
foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) {
if (!in_array($internal, $fields)) {
$fields[] = $internal;
$removal[] = $internal;
@ -255,9 +255,8 @@ class Contact
}
// Update the contact in the background if needed
$updated = max($contact['success_update'], $contact['updated'], $contact['last-update'], $contact['failure_update']);
if ((($updated < DateTimeFormat::utc('now -7 days')) || empty($contact['avatar'])) &&
in_array($contact['network'], Protocol::FEDERATED)) {
$updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']);
if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED)) {
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
}
@ -1859,10 +1858,6 @@ class Contact
return false;
}
if (Contact\Relation::isDiscoverable($ret['url'])) {
Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']);
}
if (isset($ret['hide']) && is_bool($ret['hide'])) {
$ret['unsearchable'] = $ret['hide'];
}
@ -1911,6 +1906,10 @@ class Contact
if (!$update) {
self::updateContact($id, $uid, $ret['url'], ['failed' => false, 'last-update' => $updated, 'success_update' => $updated]);
if (Contact\Relation::isDiscoverable($ret['url'])) {
Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']);
}
// Update the public contact
if ($uid != 0) {
$contact = self::getByURL($ret['url'], false, ['id']);
@ -1948,6 +1947,10 @@ class Contact
self::updateContact($id, $uid, $ret['url'], $ret);
if (Contact\Relation::isDiscoverable($ret['url'])) {
Worker::add(PRIORITY_LOW, 'ContactDiscovery', $ret['url']);
}
return true;
}
@ -2705,21 +2708,24 @@ class Contact
{
$added = 0;
$updated = 0;
$unchanged = 0;
$count = 0;
foreach ($urls as $url) {
$contact = Contact::getByURL($url, false, ['id']);
$contact = Contact::getByURL($url, false, ['id', 'updated']);
if (empty($contact['id'])) {
Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
++$added;
} else {
} elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) {
Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']);
++$updated;
} else {
++$unchanged;
}
++$count;
}
return ['count' => $count, 'added' => $added, 'updated' => $updated];
return ['count' => $count, 'added' => $added, 'updated' => $updated, 'unchanged' => $unchanged];
}
/**

2
src/Module/Search/Index.php
View file

@ -156,7 +156,7 @@ class Index extends BaseSearch
}
$last_uriid = isset($_GET['last_uriid']) ? intval($_GET['last_uriid']) : 0;
Logger::info('Blubb', ['uri' => $last_uriid]);
$pager = new Pager(DI::l10n(), DI::args()->getQueryString(), $itemsPerPage);
if ($tag) {

3
src/Worker/PullDirectory.php
View file

@ -22,7 +22,6 @@
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\DI;
use Friendica\Model\Contact;
@ -65,6 +64,6 @@ class PullDirectory
$now = $contacts['now'] ?? 0;
DI::config()->set('system', 'last-directory-sync', $now);
Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'directory' => $directory]);
Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'directory' => $directory]);
}
}

56
src/Worker/UpdatePublicContacts.php
View file

@ -36,28 +36,50 @@ class UpdatePublicContacts
public static function execute()
{
$count = 0;
$last_updated = DateTimeFormat::utc('now - 1 week');
$condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?",
Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated];
$ids = [];
$base_condition = ['network' => Protocol::FEDERATED, 'uid' => 0, 'self' => false];
if (DI::config()->get('system', 'update_active_contacts')) {
$condition = DBA::mergeConditions($condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR
`id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR
`id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`))"]);
if (!DI::config()->get('system', 'update_active_contacts')) {
// Add every contact (mostly failed ones) that hadn't been updated for six months
$condition = DBA::mergeConditions($base_condition,
["`last-update` < ?", DateTimeFormat::utc('now - 6 month')]);
$ids = self::getContactsToUpdate($condition, $ids);
// Add every non failed contact that hadn't been updated for a month
$condition = DBA::mergeConditions($base_condition,
["NOT `failed` AND `last-update` < ?", DateTimeFormat::utc('now - 1 month')]);
$ids = self::getContactsToUpdate($condition, $ids);
}
$oldest_date = '';
$oldest_id = '';
$contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]);
while ($contact = DBA::fetch($contacts)) {
if (empty($oldest_id)) {
$oldest_id = $contact['id'];
$oldest_date = $contact['last-update'];
}
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
// Add every contact our system interacted with and hadn't been updated for a week
$condition = DBA::mergeConditions($base_condition, ["(`id` IN (SELECT `author-id` FROM `item`) OR
`id` IN (SELECT `owner-id` FROM `item`) OR `id` IN (SELECT `causer-id` FROM `item`) OR
`id` IN (SELECT `cid` FROM `post-tag`) OR `id` IN (SELECT `cid` FROM `user-contact`)) AND
`last-update` < ?", DateTimeFormat::utc('now - 1 week')]);
$ids = self::getContactsToUpdate($condition, $ids);
foreach ($ids as $id) {
Worker::add(PRIORITY_LOW, "UpdateContact", $id);
++$count;
}
Logger::info('Initiated update for public contacts', ['interval' => $count, 'id' => $oldest_id, 'oldest' => $oldest_date]);
Logger::info('Initiated update for public contacts', ['count' => $count]);
}
/**
* Returns contact ids based on a given condition
*
* @param array $condition
* @param array $ids
* @return array contact ids
*/
private static function getContactsToUpdate(array $condition, array $ids = [])
{
$contacts = DBA::select('contact', ['id'], $condition, ['limit' => 100, 'order' => ['last-update']]);
while ($contact = DBA::fetch($contacts)) {
$ids[] = $contact['id'];
}
DBA::close($contacts);
return $ids;
}
}

4
src/Worker/UpdateServerDirectory.php
View file

@ -73,7 +73,7 @@ class UpdateServerDirectory
$result = Contact::addByUrls($urls);
Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'poco' => $gserver['poco']]);
Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'poco' => $gserver['poco']]);
}
private static function discoverMastodonDirectory(array $gserver)
@ -101,6 +101,6 @@ class UpdateServerDirectory
$result = Contact::addByUrls($urls);
Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'url' => $gserver['url']]);
Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'unchanged' => $result['unchanged'], 'url' => $gserver['url']]);
}
}