From 3f3f56db5749f9970d49753630e4973e6f304c32 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 30 Nov 2020 05:39:12 +0000 Subject: [PATCH 1/6] Poll interval function moved to feed class, first steps for delayed posts --- src/Core/Worker.php | 2 +- src/Protocol/Feed.php | 93 +++++++++++++++++++++++++++++++++---- src/Worker/PollContacts.php | 43 +++-------------- 3 files changed, 91 insertions(+), 47 deletions(-) diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 2dc3c9137b..33e66ae1c3 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -1226,7 +1226,7 @@ class Worker $priority = $run_parameter; } elseif (is_array($run_parameter)) { if (isset($run_parameter['delayed'])) { - $delayed = $run_parameter['execute']; + $delayed = $run_parameter['delayed']; } if (isset($run_parameter['priority'])) { $priority = $run_parameter['priority']; diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index dee1f204cc..b38d553245 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -107,6 +107,55 @@ class Feed } } + /** + * Get the poll interval for the given contact array + * + * @param array $contact + * @return int Poll interval in minutes + */ + public static function getPollInterval(array $contact) + { + if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { + $ratings = [0, 3, 7, 8, 9, 10]; + if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { + $rating = $contact['rating']; + } elseif (array_key_exists($contact['priority'], $ratings)) { + $rating = $ratings[$contact['priority']]; + } else { + $rating = -1; + } + } else { + // Check once a week per default for all other networks + $rating = 9; + } + + // Friendica and OStatus are checked once a day + if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { + $rating = 8; + } + + // Check archived contacts or contacts with unsupported protocols once a month + if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { + $rating = 10; + } + + if ($rating < 0) { + return 0; + } + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + + $min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval')); + + $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200]; + + //$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', + // '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; + + return $poll_intervals[$rating]; + } + /** * Read a RSS/RDF/Atom feed and create an item entry for it * @@ -311,6 +360,8 @@ class Feed $total_items = $max_items; } + $postings = []; + // Importing older entries first for ($i = $total_items; $i >= 0; --$i) { $entry = $entries->item($i); @@ -608,18 +659,42 @@ class Feed $notify = PRIORITY_MEDIUM; } - $id = Item::insert($item, $notify); + $postings[] = ['item' => $item, 'notify' => $notify, + 'taglist' => $taglist, 'attachments' => $attachments]; + } - Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + if (!empty($postings)) { + $total = count($postings); + if ($total > 1) { + $interval = self::getPollInterval($contact); + $delay = round(($interval * 60) / $total); + Logger::info('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); + } else { + $delay = 0; + } - if (!empty($id) && (!empty($taglist) || !empty($attachments))) { - $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); - foreach ($taglist as $tag) { - Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + $post_delay = 0; + + foreach ($postings as $posting) { + if ($delay > 0) { + $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' minute'); + Logger::info('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]); + $post_delay += $delay; } - foreach ($attachments as $attachment) { - $attachment['uri-id'] = $feeditem['uri-id']; - Post\Media::insert($attachment); + + $id = Item::insert($posting['item'], $posting['notify']); + + Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + + if (!empty($id) && (!empty($posting['taglist']) || !empty($posting['attachments']))) { + $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); + foreach ($posting['taglist'] as $tag) { + Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + } + foreach ($posting['attachments'] as $attachment) { + $attachment['uri-id'] = $feeditem['uri-id']; + Post\Media::insert($attachment); + } } } } diff --git a/src/Worker/PollContacts.php b/src/Worker/PollContacts.php index 190fa01e30..078cf202a5 100644 --- a/src/Worker/PollContacts.php +++ b/src/Worker/PollContacts.php @@ -26,7 +26,7 @@ use Friendica\Core\Protocol; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Model\Contact; +use Friendica\Protocol\Feed; use Friendica\Util\DateTimeFormat; /** @@ -58,47 +58,16 @@ class PollContacts } while ($contact = DBA::fetch($contacts)) { - if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { - $ratings = [0, 3, 7, 8, 9, 10]; - if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { - $rating = $contact['rating']; - } elseif (array_key_exists($contact['priority'], $ratings)) { - $rating = $ratings[$contact['priority']]; - } else { - $rating = -1; - } - } else { - // Check once a week per default for all other networks - $rating = 9; - } - - // Friendica and OStatus are checked once a day - if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - $rating = 8; - } - - // Check archived contacts or contacts with unsupported protocols once a month - if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { - $rating = 10; - } - - if ($rating < 0) { + $interval = Feed::getPollInterval($contact); + if ($interval == 0) { continue; } - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - - $min_poll_interval = DI::config()->get('system', 'min_poll_interval'); - - $poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', - '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; $now = DateTimeFormat::utcNow(); - $next_update = DateTimeFormat::utc($contact['last-update'] . ' + ' . $poll_intervals[$rating]); + $next_update = DateTimeFormat::utc($contact['last-update'] . ' + ' . $interval . ' minute'); - if (empty($poll_intervals[$rating]) || ($now < $next_update)) { - Logger::debug('No update', ['cid' => $contact['id'], 'rating' => $rating, 'next' => $next_update, 'now' => $now]); + if ($now < $next_update) { + Logger::debug('No update', ['cid' => $contact['id'], 'interval' => $interval, 'next' => $next_update, 'now' => $now]); continue; } From 4c39c88b52231e6c060be2c09d288495527ca1aa Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 30 Nov 2020 07:06:43 +0100 Subject: [PATCH 2/6] Adjusted log levels, removed spaces --- src/Protocol/Feed.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index b38d553245..6b46d7ac07 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -667,8 +667,8 @@ class Feed $total = count($postings); if ($total > 1) { $interval = self::getPollInterval($contact); - $delay = round(($interval * 60) / $total); - Logger::info('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); + $delay = round(($interval * 60) / $total); + Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); } else { $delay = 0; } @@ -678,13 +678,13 @@ class Feed foreach ($postings as $posting) { if ($delay > 0) { $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' minute'); - Logger::info('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]); + Logger::notice('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]); $post_delay += $delay; } $id = Item::insert($posting['item'], $posting['notify']); - Logger::info("Feed for contact " . $contact["url"] . " stored under id " . $id); + Logger::notice("Feed for contact " . $contact["url"] . " stored under id " . $id); if (!empty($id) && (!empty($posting['taglist']) || !empty($posting['attachments']))) { $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); @@ -692,7 +692,7 @@ class Feed Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); } foreach ($posting['attachments'] as $attachment) { - $attachment['uri-id'] = $feeditem['uri-id']; + $attachment['uri-id'] = $feeditem['uri-id']; Post\Media::insert($attachment); } } @@ -750,7 +750,7 @@ class Feed $oldest = $day; $oldest_date = $date; } - + if ($newest < $day) { $newest = $day; $newest_date = $date; @@ -837,7 +837,7 @@ class Feed if ($tagstr != "") { $tagstr .= ", "; } - + $tagstr .= "#[url=" . DI::baseUrl() . "/search?tag=" . urlencode($tag) . "]" . $tag . "[/url]"; } From a7e0a1f7de82bc8f0919c2ea61258a3ae09e4eba Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 30 Nov 2020 06:19:10 +0000 Subject: [PATCH 3/6] The maximum delay should be a day --- src/Protocol/Feed.php | 101 +++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 6b46d7ac07..ed3598e641 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -107,55 +107,6 @@ class Feed } } - /** - * Get the poll interval for the given contact array - * - * @param array $contact - * @return int Poll interval in minutes - */ - public static function getPollInterval(array $contact) - { - if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { - $ratings = [0, 3, 7, 8, 9, 10]; - if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { - $rating = $contact['rating']; - } elseif (array_key_exists($contact['priority'], $ratings)) { - $rating = $ratings[$contact['priority']]; - } else { - $rating = -1; - } - } else { - // Check once a week per default for all other networks - $rating = 9; - } - - // Friendica and OStatus are checked once a day - if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - $rating = 8; - } - - // Check archived contacts or contacts with unsupported protocols once a month - if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { - $rating = 10; - } - - if ($rating < 0) { - return 0; - } - /* - * Based on $contact['priority'], should we poll this site now? Or later? - */ - - $min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval')); - - $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200]; - - //$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', - // '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; - - return $poll_intervals[$rating]; - } - /** * Read a RSS/RDF/Atom feed and create an item entry for it * @@ -666,7 +617,8 @@ class Feed if (!empty($postings)) { $total = count($postings); if ($total > 1) { - $interval = self::getPollInterval($contact); + // Posts shouldn't be delayed more than a day + $interval = max(1440, self::getPollInterval($contact)); $delay = round(($interval * 60) / $total); Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); } else { @@ -823,6 +775,55 @@ class Feed } } + /** + * Get the poll interval for the given contact array + * + * @param array $contact + * @return int Poll interval in minutes + */ + public static function getPollInterval(array $contact) + { + if (in_array($contact['network'], [Protocol::MAIL, Protocol::FEED])) { + $ratings = [0, 3, 7, 8, 9, 10]; + if (DI::config()->get('system', 'adjust_poll_frequency') && ($contact['network'] == Protocol::FEED)) { + $rating = $contact['rating']; + } elseif (array_key_exists($contact['priority'], $ratings)) { + $rating = $ratings[$contact['priority']]; + } else { + $rating = -1; + } + } else { + // Check once a week per default for all other networks + $rating = 9; + } + + // Friendica and OStatus are checked once a day + if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { + $rating = 8; + } + + // Check archived contacts or contacts with unsupported protocols once a month + if ($contact['archive'] || in_array($contact['network'], [Protocol::ZOT, Protocol::PHANTOM])) { + $rating = 10; + } + + if ($rating < 0) { + return 0; + } + /* + * Based on $contact['priority'], should we poll this site now? Or later? + */ + + $min_poll_interval = max(1, DI::config()->get('system', 'min_poll_interval')); + + $poll_intervals = [$min_poll_interval, 15, 30, 60, 120, 180, 360, 720 ,1440, 10080, 43200]; + + //$poll_intervals = [$min_poll_interval . ' minute', '15 minute', '30 minute', + // '1 hour', '2 hour', '3 hour', '6 hour', '12 hour' ,'1 day', '1 week', '1 month']; + + return $poll_intervals[$rating]; + } + /** * Convert a tag array to a tag string * From cbd1ba8da2f89282d59b8a28457de7486d9f2ae4 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 30 Nov 2020 06:23:27 +0000 Subject: [PATCH 4/6] Post delay is in seconds --- src/Protocol/Feed.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index ed3598e641..eb6f71ff50 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -629,7 +629,7 @@ class Feed foreach ($postings as $posting) { if ($delay > 0) { - $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' minute'); + $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' second'); Logger::notice('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]); $post_delay += $delay; } From 6e06e0cf202e85ad28f49f69b6bf7e7a6ccd79d6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 30 Nov 2020 06:59:00 +0000 Subject: [PATCH 5/6] New class for delayed postings --- src/Protocol/Feed.php | 17 ++-------- src/Worker/DelayedPublish.php | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 src/Worker/DelayedPublish.php diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index eb6f71ff50..0c945a1809 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -29,6 +29,7 @@ use Friendica\Content\Text\HTML; use Friendica\Core\Cache\Duration; use Friendica\Core\Logger; use Friendica\Core\Protocol; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; @@ -634,20 +635,8 @@ class Feed $post_delay += $delay; } - $id = Item::insert($posting['item'], $posting['notify']); - - Logger::notice("Feed for contact " . $contact["url"] . " stored under id " . $id); - - if (!empty($id) && (!empty($posting['taglist']) || !empty($posting['attachments']))) { - $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); - foreach ($posting['taglist'] as $tag) { - Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); - } - foreach ($posting['attachments'] as $attachment) { - $attachment['uri-id'] = $feeditem['uri-id']; - Post\Media::insert($attachment); - } - } + Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $post_delay], + 'DelayedPublish', $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); } } diff --git a/src/Worker/DelayedPublish.php b/src/Worker/DelayedPublish.php new file mode 100644 index 0000000000..3dad540481 --- /dev/null +++ b/src/Worker/DelayedPublish.php @@ -0,0 +1,58 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Logger; +use Friendica\Model\Item; +use Friendica\Model\Post; +use Friendica\Model\Tag; + +class DelayedPublish +{ + /** + * Publish a post, used for delayed postings + * + * @param array $item + * @param integer $notify + * @param array $taglist + * @param array $attachments + * @return void + */ + public static function execute(array $item, int $notify = 0, array $taglist = [], array $attachments = []) + { + $id = Item::insert($item, $notify); + + Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); + + if (!empty($id) && (!empty($taglist) || !empty($attachments))) { + $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); + foreach ($taglist as $tag) { + Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + } + foreach ($attachments as $attachment) { + $attachment['uri-id'] = $feeditem['uri-id']; + Post\Media::insert($attachment); + } + } + + } +} From d0f45c4ecb192ea54313ec82dc13990a0c1041b5 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 30 Nov 2020 09:59:29 +0100 Subject: [PATCH 6/6] Fixed function name and variable --- src/Protocol/Feed.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 0c945a1809..5231107146 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -619,7 +619,7 @@ class Feed $total = count($postings); if ($total > 1) { // Posts shouldn't be delayed more than a day - $interval = max(1440, self::getPollInterval($contact)); + $interval = min(1440, self::getPollInterval($contact)); $delay = round(($interval * 60) / $total); Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); } else { @@ -635,7 +635,7 @@ class Feed $post_delay += $delay; } - Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $post_delay], + Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $publish_at], 'DelayedPublish', $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); } }