From 3f3f56db5749f9970d49753630e4973e6f304c32 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 30 Nov 2020 05:39:12 +0000 Subject: [PATCH] Poll interval function moved to feed class, first steps for delayed posts --- src/Core/Worker.php | 2 +- src/Protocol/Feed.php | 93 +++++++++++++++++++++++++++++++++---- src/Worker/PollContacts.php | 43 +++-------------- 3 files changed, 91 insertions(+), 47 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 2dc3c9137b..33e66ae1c3 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1226,7 +1226,7 @@ class Worker $priority = $run_parameter; } elseif (is_array($run_parameter)) { if (isset($run_parameter['delayed'])) { - $delayed = $run_parameter['execute']; + $delayed = $run_parameter['delayed']; } if (isset($run_parameter['priority'])) { $priority = $run_parameter['priority']; diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index dee1f204cc..b38d553245 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -107,6 +107,55 @@ class Feed } } + /** + * Get the poll interval for the given contact array + * + * @param array $contact + * @return int Poll interval in minutes + */ + public static function getPollInterval(array $contact) + { + if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { + $ratings = [0, 3, 7, 8, 9, 10]; + if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { + $rating = $contact['rating']; + } elseif (array_key_exists($contact['priority'], $ratings)) { + $rating = $ratings[$contact['priority']]; + } else { + $rating = -1; + } + } else { + // Check once a week per default for all other networks + $rating = 9; + } + + // Friendica and OStatus are checked once a day + if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { + $rating = 8; + } + + // Check archived contacts or contacts with unsupported protocols once a month + if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { + $rating = 10; + } + + if ($rating < 0) { + return 0; + } + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + + $min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval')); + + $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200]; + + //$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', + // '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; + + return $poll_intervals[$rating]; + } + /** * Read a RSS/RDF/Atom feed and create an item entry for it * @@ -311,6 +360,8 @@ class Feed $total_items = $max_items; } + $postings = []; + // Importing older entries first for ($i = $total_items; $i >= 0; --$i) { $entry = $entries->item($i); @@ -608,18 +659,42 @@ class Feed $notify = PRIORITY_MEDIUM; } - $id = Item::insert($item, $notify); + $postings[] = ['item' => $item, 'notify' => $notify, + 'taglist' => $taglist, 'attachments' => $attachments]; + } - Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + if (!empty($postings)) { + $total = count($postings); + if ($total > 1) { + $interval = self::getPollInterval($contact); + $delay = round(($interval * 60) / $total); + Logger::info('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); + } else { + $delay = 0; + } - if (!empty($id) && (!empty($taglist) || !empty($attachments))) { - $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); - foreach ($taglist as $tag) { - Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + $post_delay = 0; + + foreach ($postings as $posting) { + if ($delay > 0) { + $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' minute'); + Logger::info('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]); + $post_delay += $delay; } - foreach ($attachments as $attachment) { - $attachment['uri-id'] = $feeditem['uri-id']; - Post\Media::insert($attachment); + + $id = Item::insert($posting['item'], $posting['notify']); + + Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + + if (!empty($id) && (!empty($posting['taglist']) || !empty($posting['attachments']))) { + $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); + foreach ($posting['taglist'] as $tag) { + Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + } + foreach ($posting['attachments'] as $attachment) { + $attachment['uri-id'] = $feeditem['uri-id']; + Post\Media::insert($attachment); + } } } } diff --git a/src/Worker/PollContacts.php b/src/Worker/PollContacts.php index 190fa01e30..078cf202a5 100644 --- a/src/Worker/PollContacts.php +++ b/src/Worker/PollContacts.php @@ -26,7 +26,7 @@ use Friendica\Core\Protocol; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Model\Contact; +use Friendica\Protocol\Feed; use Friendica\Util\DateTimeFormat; /** @@ -58,47 +58,16 @@ class PollContacts } while ($contact = DBA::fetch($contacts)) { - if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { - $ratings = [0, 3, 7, 8, 9, 10]; - if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { - $rating = $contact['rating']; - } elseif (array_key_exists($contact['priority'], $ratings)) { - $rating = $ratings[$contact['priority']]; - } else { - $rating = -1; - } - } else { - // Check once a week per default for all other networks - $rating = 9; - } - - // Friendica and OStatus are checked once a day - if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - $rating = 8; - } - - // Check archived contacts or contacts with unsupported protocols once a month - if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { - $rating = 10; - } - - if ($rating < 0) { + $interval = Feed::getPollInterval($contact); + if ($interval == 0) { continue; } - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - - $min_poll_interval = DI::config()->get('system', 'min_poll_interval'); - - $poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', - '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; $now = DateTimeFormat::utcNow(); - $next_update = DateTimeFormat::utc($contact['last-update'] . ' + ' . $poll_intervals[$rating]); + $next_update = DateTimeFormat::utc($contact['last-update'] . ' + ' . $interval . ' minute'); - if (empty($poll_intervals[$rating]) || ($now < $next_update)) { - Logger::debug('No update', ['cid' => $contact['id'], 'rating' => $rating, 'next' => $next_update, 'now' => $now]); + if ($now < $next_update) { + Logger::debug('No update', ['cid' => $contact['id'], 'interval' => $interval, 'next' => $next_update, 'now' => $now]); continue; }