From 2aa302c570ca75969ad0ed4697d4f38cf98741ab Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Dec 2020 22:11:29 +0000 Subject: [PATCH 01/16] New class for handling delayed posts --- src/Model/Post/Delayed.php | 112 ++++++++++++++++++++++++++++++++++ src/Protocol/Feed.php | 30 ++++++--- src/Worker/DelayedPublish.php | 17 +----- static/dbstructure.config.php | 15 ++++- 4 files changed, 149 insertions(+), 25 deletions(-) create mode 100644 src/Model/Post/Delayed.php diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php new file mode 100644 index 0000000000..ccb678d466 --- /dev/null +++ b/src/Model/Post/Delayed.php @@ -0,0 +1,112 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Core\Logger; +use Friendica\Database\DBA; +use Friendica\Core\Worker; +use Friendica\Database\Database; +use Friendica\Model\Item; +use Friendica\Model\Tag; + +class Delayed +{ + /** + * Insert a new delayed post + * + * @param string $uri + * @param integer $uid + * @param string $delayed + * @param array $item + * @param integer $notify + * @param array $taglist + * @param array $attachments + * @return bool insert success + */ + public static function add(string $uri, int $uid, string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = []) + { + if (self::exists($uri)) { + return false; + } + + Logger::notice('Adding post for delayed publishing', ['uid' => $uid, 'delayed' => $delayed, 'uri' => $uri]); + + Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments); + return DBA::insert('delayed-post', ['uri' => $uri, 'uid' => $uid, 'delayed' => $delayed], Database::INSERT_IGNORE); + } + + /** + * Delete a delayed post + * + * @param string $uri + * + * @return bool delete success + */ + private static function delete(string $uri) + { + return DBA::delete('delayed-post', ['uri' => $uri]); + } + + /** + * Undocumented function + * + * @param string $uri + * + * @return bool "true" if an entry with that URI exists + */ + public static function exists(string $uri) + { + return DBA::exists('delayed-post', ['uri' => $uri]); + } + + /** + * Publish a delayed post + * + * @param array $item + * @param integer $notify + * @param array $taglist + * @param array $attachments + * @return bool + */ + public static function publish(array $item, int $notify = 0, array $taglist = [], array $attachments = []) + { + $id = Item::insert($item, $notify); + + Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); + + if (!empty($id) && (!empty($taglist) || !empty($attachments))) { + $feeditem = Item::selectFirst(['uri-id', 'uri'], ['id' => $id]); + self::delete($feeditem['uri']); + + foreach ($taglist as $tag) { + Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); + } + + foreach ($attachments as $attachment) { + $attachment['uri-id'] = $feeditem['uri-id']; + Media::insert($attachment); + } + } + + return $id; + } +} diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 6390fab658..f874b4b026 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -606,7 +606,7 @@ class Feed // Additionally we have to avoid conflicts with identical URI between imported feeds and these items. if ($notify) { $item['guid'] = Item::guidFromUri($orig_plink, DI::baseUrl()->getHostname()); - unset($item['uri']); + $item['uri'] = Item::newURI($item['uid'], $item['guid']); unset($item['thr-parent']); unset($item['parent-uri']); @@ -614,16 +614,21 @@ class Feed $notify = PRIORITY_MEDIUM; } - $postings[] = ['item' => $item, 'notify' => $notify, - 'taglist' => $taglist, 'attachments' => $attachments]; + if (!Post\Delayed::exists($item["uri"])) { + $postings[] = ['item' => $item, 'notify' => $notify, + 'taglist' => $taglist, 'attachments' => $attachments]; + } else { + Logger::info('Post already exists in the delayed posts queue', ['uri' => $item["uri"]]); + } } if (!empty($postings)) { + $min_posting = DI::config()->get('system', 'minimum_posting_interval', 0); $total = count($postings); if ($total > 1) { // Posts shouldn't be delayed more than a day $interval = min(1440, self::getPollInterval($contact)); - $delay = round(($interval * 60) / $total); + $delay = max(round(($interval * 60) / $total), 60 * $min_posting); Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); } else { $delay = 0; @@ -633,15 +638,22 @@ class Feed foreach ($postings as $posting) { if ($delay > 0) { - $publish_at = DateTimeFormat::utc('now + ' . $post_delay . ' second'); - Logger::notice('Got publishing date', ['delay' => $delay, 'publish_at' => $publish_at, 'cid' => $contact['id'], 'url' => $contact['url']]); + $publish_time = time() + $post_delay; + Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); $post_delay += $delay; } else { - $publish_at = DBA::NULL_DATETIME; + $publish_time = time(); } - Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $publish_at], - 'DelayedPublish', $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); + $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); + $next_publish = max($last_publish + (60 * $min_posting), time()); + if ($publish_time < $next_publish) { + $publish_time = $next_publish; + } + $publish_at = date(DateTimeFormat::MYSQL, $publish_time); + + Post\Delayed::add($item['uri'], $item['uid'], $publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); + DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish); } } diff --git a/src/Worker/DelayedPublish.php b/src/Worker/DelayedPublish.php index 3dad540481..beffb22e30 100644 --- a/src/Worker/DelayedPublish.php +++ b/src/Worker/DelayedPublish.php @@ -39,20 +39,7 @@ class DelayedPublish */ public static function execute(array $item, int $notify = 0, array $taglist = [], array $attachments = []) { - $id = Item::insert($item, $notify); - - Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); - - if (!empty($id) && (!empty($taglist) || !empty($attachments))) { - $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); - foreach ($taglist as $tag) { - Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); - } - foreach ($attachments as $attachment) { - $attachment['uri-id'] = $feeditem['uri-id']; - Post\Media::insert($attachment); - } - } - + $id = Post\Delayed::publish($item, $notify, $taglist, $attachments); + Logger::notice('Post published', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 9a125cc15b..65b4fecf08 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1381); + define('DB_UPDATE_VERSION', 1382); } return [ @@ -527,6 +527,19 @@ return [ "received" => ["received"], ] ], + "delayed-post" => [ + "comment" => "Posts that are about to be posted at a later time", + "fields" => [ + "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"], + "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be posted later"], + "uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"], + "delayed" => ["type" => "datetime", "comment" => "delay time"], + ], + "indexes" => [ + "PRIMARY" => ["id"], + "url" => ["UNIQUE", "url"], + ] + ], "diaspora-interaction" => [ "comment" => "Signed Diaspora Interaction", "fields" => [ From 61ecdf789c648cc5789ade2da3172bb7cc7a734d Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Dec 2020 22:21:44 +0000 Subject: [PATCH 02/16] Fix table structure --- database.sql | 32 +++++++++++++++++++++++--------- static/dbstructure.config.php | 2 +- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/database.sql b/database.sql index 8564a8ba02..520c5e9b05 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.12-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1381 +-- DB_UPDATE_VERSION 1382 -- ------------------------------------------ @@ -461,6 +461,19 @@ CREATE TABLE IF NOT EXISTS `conversation` ( INDEX `received` (`received`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages'; +-- +-- TABLE delayed-post +-- +CREATE TABLE IF NOT EXISTS `delayed-post` ( + `id` int unsigned NOT NULL auto_increment, + `uri` varchar(255) COMMENT 'URI of the post that will be posted later', + `uid` mediumint unsigned COMMENT 'Owner User id', + `delayed` datetime COMMENT 'delay time', + PRIMARY KEY(`id`), + UNIQUE INDEX `uri` (`uri`(190)), + FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be posted at a later time'; + -- -- TABLE diaspora-interaction -- @@ -1472,7 +1485,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` ( -- -- VIEW category-view -- -DROP VIEW IF EXISTS `category-view`; +DROP VIEW `category-view`; CREATE VIEW `category-view` AS SELECT `post-category`.`uri-id` AS `uri-id`, `post-category`.`uid` AS `uid`, @@ -1489,7 +1502,7 @@ CREATE VIEW `category-view` AS SELECT -- -- VIEW tag-view -- -DROP VIEW IF EXISTS `tag-view`; +DROP VIEW `tag-view`; CREATE VIEW `tag-view` AS SELECT `post-tag`.`uri-id` AS `uri-id`, `item-uri`.`uri` AS `uri`, @@ -1507,7 +1520,7 @@ CREATE VIEW `tag-view` AS SELECT -- -- VIEW network-item-view -- -DROP VIEW IF EXISTS `network-item-view`; +DROP VIEW `network-item-view`; CREATE VIEW `network-item-view` AS SELECT `item`.`parent-uri-id` AS `uri-id`, `item`.`parent-uri` AS `uri`, @@ -1539,7 +1552,7 @@ CREATE VIEW `network-item-view` AS SELECT -- -- VIEW network-thread-view -- -DROP VIEW IF EXISTS `network-thread-view`; +DROP VIEW `network-thread-view`; CREATE VIEW `network-thread-view` AS SELECT `item`.`uri-id` AS `uri-id`, `item`.`uri` AS `uri`, @@ -1570,7 +1583,7 @@ CREATE VIEW `network-thread-view` AS SELECT -- -- VIEW owner-view -- -DROP VIEW IF EXISTS `owner-view`; +DROP VIEW `owner-view`; CREATE VIEW `owner-view` AS SELECT `contact`.`id` AS `id`, `contact`.`uid` AS `uid`, @@ -1705,7 +1718,7 @@ CREATE VIEW `owner-view` AS SELECT -- -- VIEW pending-view -- -DROP VIEW IF EXISTS `pending-view`; +DROP VIEW `pending-view`; CREATE VIEW `pending-view` AS SELECT `register`.`id` AS `id`, `register`.`hash` AS `hash`, @@ -1727,7 +1740,7 @@ CREATE VIEW `pending-view` AS SELECT -- -- VIEW tag-search-view -- -DROP VIEW IF EXISTS `tag-search-view`; +DROP VIEW `tag-search-view`; CREATE VIEW `tag-search-view` AS SELECT `post-tag`.`uri-id` AS `uri-id`, `item`.`id` AS `iid`, @@ -1748,7 +1761,7 @@ CREATE VIEW `tag-search-view` AS SELECT -- -- VIEW workerqueue-view -- -DROP VIEW IF EXISTS `workerqueue-view`; +DROP VIEW `workerqueue-view`; CREATE VIEW `workerqueue-view` AS SELECT `process`.`pid` AS `pid`, `workerqueue`.`priority` AS `priority` @@ -1756,3 +1769,4 @@ CREATE VIEW `workerqueue-view` AS SELECT INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` WHERE NOT `workerqueue`.`done`; + diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 65b4fecf08..9af9c46257 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -537,7 +537,7 @@ return [ ], "indexes" => [ "PRIMARY" => ["id"], - "url" => ["UNIQUE", "url"], + "uri" => ["UNIQUE", "uri(190)"], ] ], "diaspora-interaction" => [ From b3f71d174c2f64d4bb527a6343cc4ba88f410a42 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 1 Dec 2020 23:54:49 +0100 Subject: [PATCH 03/16] Added logging --- src/Model/Post/Delayed.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php index ccb678d466..14f5d7198e 100644 --- a/src/Model/Post/Delayed.php +++ b/src/Model/Post/Delayed.php @@ -67,7 +67,7 @@ class Delayed } /** - * Undocumented function + * Check if an entry exists * * @param string $uri * @@ -95,7 +95,8 @@ class Delayed if (!empty($id) && (!empty($taglist) || !empty($attachments))) { $feeditem = Item::selectFirst(['uri-id', 'uri'], ['id' => $id]); - self::delete($feeditem['uri']); + $result = self::delete($feeditem['uri']); + Logger::notice('Delayed post entry deleted', ['result' => $result, 'item' => $feeditem]); foreach ($taglist as $tag) { Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); From a283b1183b4ec79ac2e35c0757391c4605bf30d4 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Dec 2020 23:04:04 +0000 Subject: [PATCH 04/16] Move deletion --- src/Model/Post/Delayed.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php index 14f5d7198e..357a3e8aa9 100644 --- a/src/Model/Post/Delayed.php +++ b/src/Model/Post/Delayed.php @@ -93,10 +93,14 @@ class Delayed Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); + // It should always contain an URI since this is needed to create a delayed post entry + if (!empty($item['uri'])) { + $result = self::delete($item['uri']); + Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]); + } + if (!empty($id) && (!empty($taglist) || !empty($attachments))) { - $feeditem = Item::selectFirst(['uri-id', 'uri'], ['id' => $id]); - $result = self::delete($feeditem['uri']); - Logger::notice('Delayed post entry deleted', ['result' => $result, 'item' => $feeditem]); + $feeditem = Item::selectFirst(['uri-id'], ['id' => $id]); foreach ($taglist as $tag) { Tag::store($feeditem['uri-id'], Tag::HASHTAG, $tag); From b047d31a4f681ec3038607cb1938f47a8f81ce88 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Dec 2020 23:46:50 +0000 Subject: [PATCH 05/16] Added logging and default config value --- src/Protocol/Feed.php | 4 ++++ static/defaults.config.php | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index f874b4b026..2d5fbff4f7 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -648,6 +648,10 @@ class Feed $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); $next_publish = max($last_publish + (60 * $min_posting), time()); if ($publish_time < $next_publish) { + Logger::notice('Adapting publish time', + ['last' => date(DateTimeFormat::MYSQL, $last_publish), + 'next' => date(DateTimeFormat::MYSQL, $next_publish), + 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); $publish_time = $next_publish; } $publish_at = date(DateTimeFormat::MYSQL, $publish_time); diff --git a/static/defaults.config.php b/static/defaults.config.php index f70bf8c8f6..e7d7985aa4 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -383,6 +383,10 @@ return [ // minimal distance in minutes between two polls for a contact. Reasonable values are between 1 and 59. 'min_poll_interval' => 15, + // minimum_posting_interval (Integer) + // Minimum interval between two feed posts per user + 'minimum_posting_interval' => 0, + // no_count (Boolean) // Don't do count calculations (currently only when showing photo albums). 'no_count' => false, From 4fbd0a46b58cae1f04059b1ab9b0df654940cba7 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 00:05:03 +0000 Subject: [PATCH 06/16] Reducing parameter list --- src/Model/Post/Delayed.php | 10 ++++------ src/Protocol/Feed.php | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php index 357a3e8aa9..963122b5b8 100644 --- a/src/Model/Post/Delayed.php +++ b/src/Model/Post/Delayed.php @@ -33,8 +33,6 @@ class Delayed /** * Insert a new delayed post * - * @param string $uri - * @param integer $uid * @param string $delayed * @param array $item * @param integer $notify @@ -42,16 +40,16 @@ class Delayed * @param array $attachments * @return bool insert success */ - public static function add(string $uri, int $uid, string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = []) + public static function add(string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = []) { - if (self::exists($uri)) { + if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'])) { return false; } - Logger::notice('Adding post for delayed publishing', ['uid' => $uid, 'delayed' => $delayed, 'uri' => $uri]); + Logger::notice('Adding post for delayed publishing', ['uid' => $item['uid'], 'delayed' => $delayed, 'uri' => $item['uri']]); Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments); - return DBA::insert('delayed-post', ['uri' => $uri, 'uid' => $uid, 'delayed' => $delayed], Database::INSERT_IGNORE); + return DBA::insert('delayed-post', ['uri' => $item['uri'], 'uid' => $item['uid'], 'delayed' => $delayed], Database::INSERT_IGNORE); } /** diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 2d5fbff4f7..821de8c335 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -656,7 +656,7 @@ class Feed } $publish_at = date(DateTimeFormat::MYSQL, $publish_time); - Post\Delayed::add($item['uri'], $item['uid'], $publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); + Post\Delayed::add($publish_at, $posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish); } } From c302b7c347c43b3f7d1e997bf3447b7d48d7f9b1 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 00:20:07 +0000 Subject: [PATCH 07/16] Check for existing post --- src/Protocol/Feed.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 821de8c335..d3d0d5ef46 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -614,7 +614,8 @@ class Feed $notify = PRIORITY_MEDIUM; } - if (!Post\Delayed::exists($item["uri"])) { + $condition = ['uid' => $item['uid'], 'uri' => $item['uri'], 'network' => Protocol::FEED]; + if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"])) { $postings[] = ['item' => $item, 'notify' => $notify, 'taglist' => $taglist, 'attachments' => $attachments]; } else { From 425440ec246c6caed22bf99d6d6763c6daf50e3d Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 00:29:57 +0000 Subject: [PATCH 08/16] The network doesn't matter --- src/Protocol/Feed.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index d3d0d5ef46..48677a91b7 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -614,7 +614,7 @@ class Feed $notify = PRIORITY_MEDIUM; } - $condition = ['uid' => $item['uid'], 'uri' => $item['uri'], 'network' => Protocol::FEED]; + $condition = ['uid' => $item['uid'], 'uri' => $item['uri']]; if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"])) { $postings[] = ['item' => $item, 'notify' => $notify, 'taglist' => $taglist, 'attachments' => $attachments]; From b2dadf0f92a2caaa52bb673736183d212bb85ba3 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 00:34:10 +0000 Subject: [PATCH 09/16] Use a combined unique index --- database.sql | 2 +- src/Model/Post/Delayed.php | 12 ++++++------ src/Protocol/Feed.php | 2 +- static/dbstructure.config.php | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/database.sql b/database.sql index 520c5e9b05..99053884d8 100644 --- a/database.sql +++ b/database.sql @@ -470,7 +470,7 @@ CREATE TABLE IF NOT EXISTS `delayed-post` ( `uid` mediumint unsigned COMMENT 'Owner User id', `delayed` datetime COMMENT 'delay time', PRIMARY KEY(`id`), - UNIQUE INDEX `uri` (`uri`(190)), + UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)), FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be posted at a later time'; diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php index 963122b5b8..a29d05422a 100644 --- a/src/Model/Post/Delayed.php +++ b/src/Model/Post/Delayed.php @@ -42,7 +42,7 @@ class Delayed */ public static function add(string $delayed, array $item, int $notify = 0, array $taglist = [], array $attachments = []) { - if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'])) { + if (empty($item['uri']) || empty($item['uid']) || self::exists($item['uri'], $item['uid'])) { return false; } @@ -59,9 +59,9 @@ class Delayed * * @return bool delete success */ - private static function delete(string $uri) + private static function delete(string $uri, int $uid) { - return DBA::delete('delayed-post', ['uri' => $uri]); + return DBA::delete('delayed-post', ['uri' => $uri, 'uid' => $uid]); } /** @@ -71,9 +71,9 @@ class Delayed * * @return bool "true" if an entry with that URI exists */ - public static function exists(string $uri) + public static function exists(string $uri, int $uid) { - return DBA::exists('delayed-post', ['uri' => $uri]); + return DBA::exists('delayed-post', ['uri' => $uri, 'uid' => $uid]); } /** @@ -93,7 +93,7 @@ class Delayed // It should always contain an URI since this is needed to create a delayed post entry if (!empty($item['uri'])) { - $result = self::delete($item['uri']); + $result = self::delete($item['uri'], $item['uid']); Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]); } diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 48677a91b7..ed24defc2e 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -615,7 +615,7 @@ class Feed } $condition = ['uid' => $item['uid'], 'uri' => $item['uri']]; - if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"])) { + if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"], $item['uid'])) { $postings[] = ['item' => $item, 'notify' => $notify, 'taglist' => $taglist, 'attachments' => $attachments]; } else { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 9af9c46257..106ae4e3f0 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -537,7 +537,7 @@ return [ ], "indexes" => [ "PRIMARY" => ["id"], - "uri" => ["UNIQUE", "uri(190)"], + "uid_uri" => ["UNIQUE", "uid", "uri(190)"], ] ], "diaspora-interaction" => [ From e9fecf1541bb233b1a702ee7b00af4f539e39a5d Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 04:53:54 +0000 Subject: [PATCH 10/16] Only delay mirrored feeds --- src/Protocol/Feed.php | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index ed24defc2e..1cecee1e23 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -638,22 +638,24 @@ class Feed $post_delay = 0; foreach ($postings as $posting) { - if ($delay > 0) { - $publish_time = time() + $post_delay; - Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); - $post_delay += $delay; - } else { - $publish_time = time(); - } + $publish_time = time(); - $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); - $next_publish = max($last_publish + (60 * $min_posting), time()); - if ($publish_time < $next_publish) { - Logger::notice('Adapting publish time', - ['last' => date(DateTimeFormat::MYSQL, $last_publish), - 'next' => date(DateTimeFormat::MYSQL, $next_publish), - 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); - $publish_time = $next_publish; + if ($posting['notify']) { + if ($delay > 0) { + $publish_time = time() + $post_delay; + Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); + $post_delay += $delay; + } + + $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); + $next_publish = max($last_publish + (60 * $min_posting), time()); + if ($publish_time < $next_publish) { + Logger::notice('Adapting publish time', + ['last' => date(DateTimeFormat::MYSQL, $last_publish), + 'next' => date(DateTimeFormat::MYSQL, $next_publish), + 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); + $publish_time = $next_publish; + } } $publish_at = date(DateTimeFormat::MYSQL, $publish_time); From 775075c5db8a06ffa0d0808fb5c45e9908cc9074 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 05:07:44 +0000 Subject: [PATCH 11/16] Only spool mirrored posts --- src/Model/Post/Delayed.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php index a29d05422a..87e5fe8235 100644 --- a/src/Model/Post/Delayed.php +++ b/src/Model/Post/Delayed.php @@ -92,7 +92,7 @@ class Delayed Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); // It should always contain an URI since this is needed to create a delayed post entry - if (!empty($item['uri'])) { + if (!empty($item['uri']) && self::exists($item['uri'], $item['uid'])) { $result = self::delete($item['uri'], $item['uid']); Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]); } From b54465ba9c3c3fcea52f2d2907496c056dc0e152 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 05:08:39 +0000 Subject: [PATCH 12/16] Directly post non mirrored feed items --- src/Protocol/Feed.php | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index 1cecee1e23..dbfb72977d 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -638,24 +638,27 @@ class Feed $post_delay = 0; foreach ($postings as $posting) { - $publish_time = time(); + if (!$posting['notify']) { + Post\Delayed::publish($posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); + continue; + } - if ($posting['notify']) { - if ($delay > 0) { - $publish_time = time() + $post_delay; - Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); - $post_delay += $delay; - } + if ($delay > 0) { + $publish_time = time() + $post_delay; + Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); + $post_delay += $delay; + } else { + $publish_time = time(); + } - $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); - $next_publish = max($last_publish + (60 * $min_posting), time()); - if ($publish_time < $next_publish) { - Logger::notice('Adapting publish time', - ['last' => date(DateTimeFormat::MYSQL, $last_publish), - 'next' => date(DateTimeFormat::MYSQL, $next_publish), - 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); - $publish_time = $next_publish; - } + $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); + $next_publish = max($last_publish + (60 * $min_posting), time()); + if ($publish_time < $next_publish) { + Logger::notice('Adapting publish time', + ['last' => date(DateTimeFormat::MYSQL, $last_publish), + 'next' => date(DateTimeFormat::MYSQL, $next_publish), + 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); + $publish_time = $next_publish; } $publish_at = date(DateTimeFormat::MYSQL, $publish_time); From 6c3b1ac4bd1a27237209e63a6ee9bc0d590563a8 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 05:41:33 +0000 Subject: [PATCH 13/16] Moved direct storage --- src/Protocol/Feed.php | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index dbfb72977d..c5234c8cd4 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -616,10 +616,14 @@ class Feed $condition = ['uid' => $item['uid'], 'uri' => $item['uri']]; if (!Item::exists($condition) && !Post\Delayed::exists($item["uri"], $item['uid'])) { - $postings[] = ['item' => $item, 'notify' => $notify, - 'taglist' => $taglist, 'attachments' => $attachments]; + if (!$notify) { + Post\Delayed::publish($item, $notify, $taglist, $attachments); + } else { + $postings[] = ['item' => $item, 'notify' => $notify, + 'taglist' => $taglist, 'attachments' => $attachments]; + } } else { - Logger::info('Post already exists in the delayed posts queue', ['uri' => $item["uri"]]); + Logger::info('Post already crated or exists in the delayed posts queue', ['uri' => $item["uri"]]); } } @@ -638,11 +642,6 @@ class Feed $post_delay = 0; foreach ($postings as $posting) { - if (!$posting['notify']) { - Post\Delayed::publish($posting['item'], $posting['notify'], $posting['taglist'], $posting['attachments']); - continue; - } - if ($delay > 0) { $publish_time = time() + $post_delay; Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); @@ -657,7 +656,9 @@ class Feed Logger::notice('Adapting publish time', ['last' => date(DateTimeFormat::MYSQL, $last_publish), 'next' => date(DateTimeFormat::MYSQL, $next_publish), - 'publish' => date(DateTimeFormat::MYSQL, $publish_time)]); + 'publish' => date(DateTimeFormat::MYSQL, $publish_time), + 'uid' => $posting['item']['uid'], 'cid' => $posting['item']['contact-id'], + 'uri' => $posting['item']["uri"]]); $publish_time = $next_publish; } $publish_at = date(DateTimeFormat::MYSQL, $publish_time); From c1d1b189f7356aa221f4b9d7138e34ba5de51b17 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 08:36:23 +0000 Subject: [PATCH 14/16] Cleaned code --- src/Model/Post/Delayed.php | 4 +--- src/Protocol/Feed.php | 11 ++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/Model/Post/Delayed.php b/src/Model/Post/Delayed.php index 87e5fe8235..bfab406282 100644 --- a/src/Model/Post/Delayed.php +++ b/src/Model/Post/Delayed.php @@ -91,10 +91,8 @@ class Delayed Logger::notice('Post stored', ['id' => $id, 'uid' => $item['uid'], 'cid' => $item['contact-id']]); - // It should always contain an URI since this is needed to create a delayed post entry if (!empty($item['uri']) && self::exists($item['uri'], $item['uid'])) { - $result = self::delete($item['uri'], $item['uid']); - Logger::notice('Delayed post entry deleted', ['result' => $result, 'uri' => $item['uri']]); + self::delete($item['uri'], $item['uid']); } if (!empty($id) && (!empty($taglist) || !empty($attachments))) { diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index c5234c8cd4..fa568b4c3c 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -623,7 +623,7 @@ class Feed 'taglist' => $taglist, 'attachments' => $attachments]; } } else { - Logger::info('Post already crated or exists in the delayed posts queue', ['uri' => $item["uri"]]); + Logger::info('Post already crated or exists in the delayed posts queue', ['uid' => $item['uid'], 'uri' => $item["uri"]]); } } @@ -634,7 +634,7 @@ class Feed // Posts shouldn't be delayed more than a day $interval = min(1440, self::getPollInterval($contact)); $delay = max(round(($interval * 60) / $total), 60 * $min_posting); - Logger::notice('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); + Logger::info('Got posting delay', ['delay' => $delay, 'interval' => $interval, 'items' => $total, 'cid' => $contact['id'], 'url' => $contact['url']]); } else { $delay = 0; } @@ -644,7 +644,6 @@ class Feed foreach ($postings as $posting) { if ($delay > 0) { $publish_time = time() + $post_delay; - Logger::notice('Got publishing date', ['delay' => $delay, 'cid' => $contact['id'], 'url' => $contact['url']]); $post_delay += $delay; } else { $publish_time = time(); @@ -653,12 +652,6 @@ class Feed $last_publish = DI::pConfig()->get($posting['item']['uid'], 'system', 'last_publish', 0, true); $next_publish = max($last_publish + (60 * $min_posting), time()); if ($publish_time < $next_publish) { - Logger::notice('Adapting publish time', - ['last' => date(DateTimeFormat::MYSQL, $last_publish), - 'next' => date(DateTimeFormat::MYSQL, $next_publish), - 'publish' => date(DateTimeFormat::MYSQL, $publish_time), - 'uid' => $posting['item']['uid'], 'cid' => $posting['item']['contact-id'], - 'uri' => $posting['item']["uri"]]); $publish_time = $next_publish; } $publish_at = date(DateTimeFormat::MYSQL, $publish_time); From bd4dc10588d2c1b5d3c2b4ba2476d87700b0d4a6 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 08:50:45 +0000 Subject: [PATCH 15/16] Ensure that the database.sql contains "if exists" --- database.sql | 16 ++++++++-------- src/Database/View.php | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/database.sql b/database.sql index 99053884d8..e5e1ca2393 100644 --- a/database.sql +++ b/database.sql @@ -1485,7 +1485,7 @@ CREATE TABLE IF NOT EXISTS `workerqueue` ( -- -- VIEW category-view -- -DROP VIEW `category-view`; +DROP VIEW IF EXISTS `category-view`; CREATE VIEW `category-view` AS SELECT `post-category`.`uri-id` AS `uri-id`, `post-category`.`uid` AS `uid`, @@ -1502,7 +1502,7 @@ CREATE VIEW `category-view` AS SELECT -- -- VIEW tag-view -- -DROP VIEW `tag-view`; +DROP VIEW IF EXISTS `tag-view`; CREATE VIEW `tag-view` AS SELECT `post-tag`.`uri-id` AS `uri-id`, `item-uri`.`uri` AS `uri`, @@ -1520,7 +1520,7 @@ CREATE VIEW `tag-view` AS SELECT -- -- VIEW network-item-view -- -DROP VIEW `network-item-view`; +DROP VIEW IF EXISTS `network-item-view`; CREATE VIEW `network-item-view` AS SELECT `item`.`parent-uri-id` AS `uri-id`, `item`.`parent-uri` AS `uri`, @@ -1552,7 +1552,7 @@ CREATE VIEW `network-item-view` AS SELECT -- -- VIEW network-thread-view -- -DROP VIEW `network-thread-view`; +DROP VIEW IF EXISTS `network-thread-view`; CREATE VIEW `network-thread-view` AS SELECT `item`.`uri-id` AS `uri-id`, `item`.`uri` AS `uri`, @@ -1583,7 +1583,7 @@ CREATE VIEW `network-thread-view` AS SELECT -- -- VIEW owner-view -- -DROP VIEW `owner-view`; +DROP VIEW IF EXISTS `owner-view`; CREATE VIEW `owner-view` AS SELECT `contact`.`id` AS `id`, `contact`.`uid` AS `uid`, @@ -1718,7 +1718,7 @@ CREATE VIEW `owner-view` AS SELECT -- -- VIEW pending-view -- -DROP VIEW `pending-view`; +DROP VIEW IF EXISTS `pending-view`; CREATE VIEW `pending-view` AS SELECT `register`.`id` AS `id`, `register`.`hash` AS `hash`, @@ -1740,7 +1740,7 @@ CREATE VIEW `pending-view` AS SELECT -- -- VIEW tag-search-view -- -DROP VIEW `tag-search-view`; +DROP VIEW IF EXISTS `tag-search-view`; CREATE VIEW `tag-search-view` AS SELECT `post-tag`.`uri-id` AS `uri-id`, `item`.`id` AS `iid`, @@ -1761,7 +1761,7 @@ CREATE VIEW `tag-search-view` AS SELECT -- -- VIEW workerqueue-view -- -DROP VIEW `workerqueue-view`; +DROP VIEW IF EXISTS `workerqueue-view`; CREATE VIEW `workerqueue-view` AS SELECT `process`.`pid` AS `pid`, `workerqueue`.`priority` AS `priority` diff --git a/src/Database/View.php b/src/Database/View.php index 51d051786d..fd0a8ce1a5 100644 --- a/src/Database/View.php +++ b/src/Database/View.php @@ -112,9 +112,9 @@ class View } if (self::isView($name)) { - $sql = sprintf("DROP VIEW `%s`", DBA::escape($name)); + $sql = sprintf("DROP VIEW IF EXISTS `%s`", DBA::escape($name)); } elseif (self::isTable($name)) { - $sql = sprintf("DROP TABLE `%s`", DBA::escape($name)); + $sql = sprintf("DROP TABLE IF EXISTS `%s`", DBA::escape($name)); } if ($verbose) { From 90349d14e1e76e6a60506789299cd309602934f3 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Dec 2020 14:55:57 +0000 Subject: [PATCH 16/16] Chaning wording --- database.sql | 4 ++-- static/dbstructure.config.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/database.sql b/database.sql index e5e1ca2393..b04b088d8d 100644 --- a/database.sql +++ b/database.sql @@ -466,13 +466,13 @@ CREATE TABLE IF NOT EXISTS `conversation` ( -- CREATE TABLE IF NOT EXISTS `delayed-post` ( `id` int unsigned NOT NULL auto_increment, - `uri` varchar(255) COMMENT 'URI of the post that will be posted later', + `uri` varchar(255) COMMENT 'URI of the post that will be distributed later', `uid` mediumint unsigned COMMENT 'Owner User id', `delayed` datetime COMMENT 'delay time', PRIMARY KEY(`id`), UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)), FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be posted at a later time'; +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be distributed at a later time'; -- -- TABLE diaspora-interaction diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 106ae4e3f0..bc07c67245 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -528,10 +528,10 @@ return [ ] ], "delayed-post" => [ - "comment" => "Posts that are about to be posted at a later time", + "comment" => "Posts that are about to be distributed at a later time", "fields" => [ "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"], - "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be posted later"], + "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be distributed later"], "uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"], "delayed" => ["type" => "datetime", "comment" => "delay time"], ],