Merge pull request #8443 from annando/loops

Prevent endless loops and long running feed processing
This commit is contained in:
Hypolite Petovan 2020-03-22 11:26:13 -04:00 committed by GitHub
commit 3b3085369c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 16 additions and 51 deletions

View file

@ -884,11 +884,11 @@ class GContact
$items = $outbox['orderedItems']; $items = $outbox['orderedItems'];
} elseif (!empty($outbox['first']['orderedItems'])) { } elseif (!empty($outbox['first']['orderedItems'])) {
$items = $outbox['first']['orderedItems']; $items = $outbox['first']['orderedItems'];
} elseif (!empty($outbox['first']['href'])) { } elseif (!empty($outbox['first']['href']) && ($outbox['first']['href'] != $feed)) {
self::updateFromOutbox($outbox['first']['href'], $data); self::updateFromOutbox($outbox['first']['href'], $data);
return; return;
} elseif (!empty($outbox['first'])) { } elseif (!empty($outbox['first'])) {
if (is_string($outbox['first'])) { if (is_string($outbox['first']) && ($outbox['first'] != $feed)) {
self::updateFromOutbox($outbox['first'], $data); self::updateFromOutbox($outbox['first'], $data);
} else { } else {
Logger::warning('Unexpected data', ['outbox' => $outbox]); Logger::warning('Unexpected data', ['outbox' => $outbox]);

View file

@ -231,7 +231,7 @@ class ActivityPub
$items = $data['orderedItems']; $items = $data['orderedItems'];
} elseif (!empty($data['first']['orderedItems'])) { } elseif (!empty($data['first']['orderedItems'])) {
$items = $data['first']['orderedItems']; $items = $data['first']['orderedItems'];
} elseif (!empty($data['first']) && is_string($data['first'])) { } elseif (!empty($data['first']) && is_string($data['first']) && ($data['first'] != $url)) {
return self::fetchItems($data['first'], $uid); return self::fetchItems($data['first'], $uid);
} else { } else {
$items = []; $items = [];

View file

@ -232,8 +232,16 @@ class Feed {
} }
$items = []; $items = [];
// Limit the number of items that are about to be fetched
$total_items = ($entries->length - 1);
$max_items = DI::config()->get('system', 'max_feed_items');
if (($max_items > 0) && ($total_items > $max_items)) {
$total_items = $max_items;
}
// Importing older entries first // Importing older entries first
for ($i = $entries->length - 1; $i >= 0; --$i) { for ($i = $total_items; $i >= 0; --$i) {
$entry = $entries->item($i); $entry = $entries->item($i);
$item = array_merge($header, $author); $item = array_merge($header, $author);

View file

@ -69,9 +69,6 @@ class Cron
// Clear cache entries // Clear cache entries
Worker::add(PRIORITY_LOW, "CronJobs", "clear_cache"); Worker::add(PRIORITY_LOW, "CronJobs", "clear_cache");
// Repair missing Diaspora values in contacts
Worker::add(PRIORITY_LOW, "CronJobs", "repair_diaspora");
// Repair entries in the database // Repair entries in the database
Worker::add(PRIORITY_LOW, "CronJobs", "repair_database"); Worker::add(PRIORITY_LOW, "CronJobs", "repair_database");

View file

@ -84,10 +84,6 @@ class CronJobs
self::clearCache($a); self::clearCache($a);
break; break;
case 'repair_diaspora':
self::repairDiaspora($a);
break;
case 'repair_database': case 'repair_database':
self::repairDatabase(); self::repairDatabase();
break; break;
@ -244,46 +240,6 @@ class CronJobs
DI::config()->set('system', 'cache_last_cleared', time()); DI::config()->set('system', 'cache_last_cleared', time());
} }
/**
* Repair missing values in Diaspora contacts
*
* @param App $a
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
private static function repairDiaspora(App $a)
{
$starttime = time();
$r = q("SELECT `id`, `url` FROM `contact`
WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '')
ORDER BY RAND() LIMIT 50", DBA::escape(Protocol::DIASPORA));
if (!DBA::isResult($r)) {
return;
}
foreach ($r as $contact) {
// Quit the loop after 3 minutes
if (time() > ($starttime + 180)) {
return;
}
if (!GServer::reachable($contact["url"])) {
continue;
}
$data = Probe::uri($contact["url"]);
if ($data["network"] != Protocol::DIASPORA) {
continue;
}
Logger::log("Repair contact " . $contact["id"] . " " . $contact["url"], Logger::DEBUG);
q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d",
DBA::escape($data["batch"]), DBA::escape($data["notify"]), DBA::escape($data["poll"]), DBA::escape($data["pubkey"]),
intval($contact["id"]));
}
}
/** /**
* Do some repairs in database entries * Do some repairs in database entries
* *

View file

@ -273,6 +273,10 @@ return [
// Maximum number of queue items for a single contact before subsequent messages are discarded. // Maximum number of queue items for a single contact before subsequent messages are discarded.
'max_contact_queue' => 500, 'max_contact_queue' => 500,
// max_feed_items (Integer)
// Maximum number of feed items that are fetched and processed. For unlimited items set to 0.
'max_feed_items' => 20,
// max_image_length (Integer) // max_image_length (Integer)
// An alternate way of limiting picture upload sizes. // An alternate way of limiting picture upload sizes.
// Specify the maximum pixel length that pictures are allowed to be (for non-square pictures, it will apply to the longest side). // Specify the maximum pixel length that pictures are allowed to be (for non-square pictures, it will apply to the longest side).