From bc069c8ebfb31c7585fb0da071d71ec61d9b0e4a Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 22 Mar 2020 13:05:35 +0000 Subject: [PATCH 1/3] Prevent endless loops and long running feed processing --- src/Model/GContact.php | 4 ++-- src/Protocol/ActivityPub.php | 2 +- src/Protocol/Feed.php | 10 +++++++++- static/defaults.config.php | 4 ++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 2606edb9e..09eeda4a2 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -884,11 +884,11 @@ class GContact $items = $outbox['orderedItems']; } elseif (!empty($outbox['first']['orderedItems'])) { $items = $outbox['first']['orderedItems']; - } elseif (!empty($outbox['first']['href'])) { + } elseif (!empty($outbox['first']['href']) && ($outbox['first']['href'] != $feed)) { self::updateFromOutbox($outbox['first']['href'], $data); return; } elseif (!empty($outbox['first'])) { - if (is_string($outbox['first'])) { + if (is_string($outbox['first']) && ($outbox['first'] != $feed)) { self::updateFromOutbox($outbox['first'], $data); } else { Logger::warning('Unexpected data', ['outbox' => $outbox]); diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index 894c7f6d3..6f835e7e8 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -231,7 +231,7 @@ class ActivityPub $items = $data['orderedItems']; } elseif (!empty($data['first']['orderedItems'])) { $items = $data['first']['orderedItems']; - } elseif (!empty($data['first']) && is_string($data['first'])) { + } elseif (!empty($data['first']) && is_string($data['first']) && ($data['first'] != $url)) { return self::fetchItems($data['first'], $uid); } else { $items = []; diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 4eb6c7294..397edf3b4 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -232,8 +232,16 @@ class Feed { } $items = []; + + // Limit the number of items that are about to be fetched + $total_items = ($entries->length - 1); + $max_items = DI::config()->get('system', 'max_feed_items'); + if (($max_items > 0) && ($total_items > $max_items)) { + $total_items = $max_items; + } + // Importing older entries first - for ($i = $entries->length - 1; $i >= 0; --$i) { + for ($i = $total_items; $i >= 0; --$i) { $entry = $entries->item($i); $item = array_merge($header, $author); diff --git a/static/defaults.config.php b/static/defaults.config.php index 110c016eb..66805a121 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -273,6 +273,10 @@ return [ // Maximum number of queue items for a single contact before subsequent messages are discarded. 'max_contact_queue' => 500, + // max_feed_items (Integer) + // Maximum number of feed items that are fetched and processed. For unlimited items set to 0. + 'max_feed_items' => 10, + // max_image_length (Integer) // An alternate way of limiting picture upload sizes. // Specify the maximum pixel length that pictures are allowed to be (for non-square pictures, it will apply to the longest side). From b68a16e07acbba6efc9a1941ab740d045ab8c54d Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 22 Mar 2020 13:12:48 +0000 Subject: [PATCH 2/3] "repair diaspora" fixed some problem long time ago. It is not needed anymore --- src/Worker/Cron.php | 3 --- src/Worker/CronJobs.php | 44 ----------------------------------------- 2 files changed, 47 deletions(-) diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 714f84059..a47193334 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -69,9 +69,6 @@ class Cron // Clear cache entries Worker::add(PRIORITY_LOW, "CronJobs", "clear_cache"); - // Repair missing Diaspora values in contacts - Worker::add(PRIORITY_LOW, "CronJobs", "repair_diaspora"); - // Repair entries in the database Worker::add(PRIORITY_LOW, "CronJobs", "repair_database"); diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php index b66358b0e..2a6c97e84 100644 --- a/src/Worker/CronJobs.php +++ b/src/Worker/CronJobs.php @@ -84,10 +84,6 @@ class CronJobs self::clearCache($a); break; - case 'repair_diaspora': - self::repairDiaspora($a); - break; - case 'repair_database': self::repairDatabase(); break; @@ -244,46 +240,6 @@ class CronJobs DI::config()->set('system', 'cache_last_cleared', time()); } - /** - * Repair missing values in Diaspora contacts - * - * @param App $a - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - * @throws \ImagickException - */ - private static function repairDiaspora(App $a) - { - $starttime = time(); - - $r = q("SELECT `id`, `url` FROM `contact` - WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') - ORDER BY RAND() LIMIT 50", DBA::escape(Protocol::DIASPORA)); - if (!DBA::isResult($r)) { - return; - } - - foreach ($r as $contact) { - // Quit the loop after 3 minutes - if (time() > ($starttime + 180)) { - return; - } - - if (!GServer::reachable($contact["url"])) { - continue; - } - - $data = Probe::uri($contact["url"]); - if ($data["network"] != Protocol::DIASPORA) { - continue; - } - - Logger::log("Repair contact " . $contact["id"] . " " . $contact["url"], Logger::DEBUG); - q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", - DBA::escape($data["batch"]), DBA::escape($data["notify"]), DBA::escape($data["poll"]), DBA::escape($data["pubkey"]), - intval($contact["id"])); - } - } - /** * Do some repairs in database entries * From 771fe881ef4f9ee0ab395f8d08058861537898d6 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 22 Mar 2020 15:22:48 +0000 Subject: [PATCH 3/3] Use 20 maximum feed items as default --- static/defaults.config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/defaults.config.php b/static/defaults.config.php index 66805a121..d0725aa9b 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -275,7 +275,7 @@ return [ // max_feed_items (Integer) // Maximum number of feed items that are fetched and processed. For unlimited items set to 0. - 'max_feed_items' => 10, + 'max_feed_items' => 20, // max_image_length (Integer) // An alternate way of limiting picture upload sizes.