From 098ba4a667bb5d4d8563f096f408c282c7cdf735 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 2 May 2020 19:34:02 +0000 Subject: [PATCH 1/7] "item-delivery-data" is now "post-delivery-data" --- database.sql | 62 +++++++++++------ src/Model/Item.php | 23 ++++--- .../DeliveryData.php} | 68 +++++++++---------- src/Protocol/Diaspora.php | 6 +- src/Worker/APDelivery.php | 10 ++- src/Worker/Delivery.php | 52 +++++++------- src/Worker/Notifier.php | 8 +-- static/dbstructure.config.php | 38 +++++------ 8 files changed, 147 insertions(+), 120 deletions(-) rename src/Model/{ItemDeliveryData.php => Post/DeliveryData.php} (63%) diff --git a/database.sql b/database.sql index a68f3d4e5..320ef38e7 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.06-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1341 +-- DB_UPDATE_VERSION 1343 -- ------------------------------------------ @@ -666,7 +666,8 @@ CREATE TABLE IF NOT EXISTS `item` ( INDEX `uid_eventid` (`uid`,`event-id`), INDEX `icid` (`icid`), INDEX `iaid` (`iaid`), - INDEX `psid_wall` (`psid`,`wall`) + INDEX `psid_wall` (`psid`,`wall`), + INDEX `uri-id` (`uri-id`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts'; -- @@ -714,24 +715,6 @@ CREATE TABLE IF NOT EXISTS `item-content` ( INDEX `uri-id` (`uri-id`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts'; --- --- TABLE item-delivery-data --- -CREATE TABLE IF NOT EXISTS `item-delivery-data` ( - `iid` int unsigned NOT NULL COMMENT 'Item id', - `postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery', - `inform` mediumtext COMMENT 'Additional receivers of the linked item', - `queue_count` mediumint NOT NULL DEFAULT 0 COMMENT 'Initial number of delivery recipients, used as item.delivery_queue_count', - `queue_done` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries, used as item.delivery_queue_done', - `queue_failed` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of unsuccessful deliveries, used as item.delivery_queue_failed', - `activitypub` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via ActivityPub', - `dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via DFRN', - `legacy_dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via legacy DFRN', - `diaspora` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via Diaspora', - `ostatus` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via OStatus', - PRIMARY KEY(`iid`) -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items'; - -- -- TABLE item-uri -- @@ -1187,6 +1170,24 @@ CREATE TABLE IF NOT EXISTS `tag` ( INDEX `url` (`url`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='tags and mentions'; +-- +-- TABLE post-delivery-data +-- +CREATE TABLE IF NOT EXISTS `post-delivery-data` ( + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery', + `inform` mediumtext COMMENT 'Additional receivers of the linked item', + `queue_count` mediumint NOT NULL DEFAULT 0 COMMENT 'Initial number of delivery recipients, used as item.delivery_queue_count', + `queue_done` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries, used as item.delivery_queue_done', + `queue_failed` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of unsuccessful deliveries, used as item.delivery_queue_failed', + `activitypub` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via ActivityPub', + `dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via DFRN', + `legacy_dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via legacy DFRN', + `diaspora` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via Diaspora', + `ostatus` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via OStatus', + PRIMARY KEY(`uri-id`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items'; + -- -- TABLE post-tag -- @@ -1577,6 +1578,27 @@ CREATE VIEW `pending-view` AS SELECT INNER JOIN `contact` ON `register`.`uid` = `contact`.`uid` INNER JOIN `user` ON `register`.`uid` = `user`.`uid`; +-- +-- VIEW tag-search-view +-- +DROP VIEW IF EXISTS `tag-search-view`; +CREATE VIEW `tag-search-view` AS SELECT + `post-tag`.`uri-id` AS `uri-id`, + `item`.`id` AS `iid`, + `item`.`uri` AS `uri`, + `item`.`guid` AS `guid`, + `item`.`uid` AS `uid`, + `item`.`private` AS `private`, + `item`.`wall` AS `wall`, + `item`.`origin` AS `origin`, + `item`.`gravity` AS `gravity`, + `item`.`received` AS `received`, + `tag`.`name` AS `name` + FROM `post-tag` + INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` + INNER JOIN `item` ON `item`.`uri-id` = `post-tag`.`uri-id` + WHERE `post-tag`.`type` = 1; + -- -- VIEW workerqueue-view -- diff --git a/src/Model/Item.php b/src/Model/Item.php index 1096c2213..c01535ab6 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -283,7 +283,7 @@ class Item // Fetch data from the item-content table whenever there is content there if (self::isLegacyMode()) { - $legacy_fields = array_merge(ItemDeliveryData::LEGACY_FIELD_LIST, self::MIXED_CONTENT_FIELDLIST); + $legacy_fields = array_merge(Post\DeliveryData::LEGACY_FIELD_LIST, self::MIXED_CONTENT_FIELDLIST); foreach ($legacy_fields as $field) { if (empty($row[$field]) && !empty($row['internal-item-' . $field])) { $row[$field] = $row['internal-item-' . $field]; @@ -687,7 +687,7 @@ class Item $fields['item-content'] = array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST); - $fields['item-delivery-data'] = array_merge(ItemDeliveryData::LEGACY_FIELD_LIST, ItemDeliveryData::FIELD_LIST); + $fields['post-delivery-data'] = array_merge(Post\DeliveryData::LEGACY_FIELD_LIST, Post\DeliveryData::FIELD_LIST); $fields['permissionset'] = ['allow_cid', 'allow_gid', 'deny_cid', 'deny_gid']; @@ -809,8 +809,8 @@ class Item $joins .= " LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`"; } - if (strpos($sql_commands, "`item-delivery-data`.") !== false) { - $joins .= " LEFT JOIN `item-delivery-data` ON `item-delivery-data`.`iid` = `item`.`id`"; + if (strpos($sql_commands, "`post-delivery-data`.") !== false) { + $joins .= " LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin`"; } if (strpos($sql_commands, "`permissionset`.") !== false) { @@ -850,7 +850,7 @@ class Item $selected[] = 'internal-user-ignored'; } - $legacy_fields = array_merge(ItemDeliveryData::LEGACY_FIELD_LIST, self::MIXED_CONTENT_FIELDLIST); + $legacy_fields = array_merge(Post\DeliveryData::LEGACY_FIELD_LIST, self::MIXED_CONTENT_FIELDLIST); $selection = []; foreach ($fields as $table => $table_fields) { @@ -936,7 +936,7 @@ class Item } } - $delivery_data = ItemDeliveryData::extractFields($fields); + $delivery_data = Post\DeliveryData::extractFields($fields); $clear_fields = ['bookmark', 'type', 'author-name', 'author-avatar', 'author-link', 'owner-name', 'owner-avatar', 'owner-link', 'postopts', 'inform']; foreach ($clear_fields as $field) { @@ -1038,7 +1038,7 @@ class Item } } - ItemDeliveryData::update($item['id'], $delivery_data); + Post\DeliveryData::update($item['uri-id'], $delivery_data); self::updateThread($item['id']); @@ -1118,7 +1118,7 @@ class Item { Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]); // locate item to be deleted - $fields = ['id', 'uri', 'uid', 'parent', 'parent-uri', 'origin', + $fields = ['id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri', 'origin', 'deleted', 'file', 'resource-id', 'event-id', 'attach', 'verb', 'object-type', 'object', 'target', 'contact-id', 'icid', 'iaid', 'psid']; @@ -1202,7 +1202,7 @@ class Item self::markForDeletion(['uri' => $item['uri'], 'uid' => 0, 'deleted' => false], $priority); } - ItemDeliveryData::delete($item['id']); + Post\DeliveryData::delete($item['uri-id']); // We don't delete the item-activity here, since we need some of the data for ActivityPub @@ -1903,7 +1903,8 @@ class Item self::insertContent($item); } - $delivery_data = ItemDeliveryData::extractFields($item); + $postfields = $item; + $delivery_data = Post\DeliveryData::extractFields($postfields); unset($item['postopts']); unset($item['inform']); @@ -2007,7 +2008,7 @@ class Item } if (!empty($item['origin']) || !empty($item['wall']) || !empty($delivery_data['postopts']) || !empty($delivery_data['inform'])) { - ItemDeliveryData::insert($current_post, $delivery_data); + Post\DeliveryData::insert($item['uri-id'], $delivery_data); } DBA::commit(); diff --git a/src/Model/ItemDeliveryData.php b/src/Model/Post/DeliveryData.php similarity index 63% rename from src/Model/ItemDeliveryData.php rename to src/Model/Post/DeliveryData.php index a26c80b1c..0feb38281 100644 --- a/src/Model/ItemDeliveryData.php +++ b/src/Model/Post/DeliveryData.php @@ -19,12 +19,12 @@ * */ -namespace Friendica\Model; +namespace Friendica\Model\Post; use Friendica\Database\DBA; use \BadMethodCallException; -class ItemDeliveryData +class DeliveryData { const LEGACY_FIELD_LIST = [ // Legacy fields moved from item table @@ -55,7 +55,7 @@ class ItemDeliveryData public static function extractFields(array &$fields) { $delivery_data = []; - foreach (array_merge(ItemDeliveryData::FIELD_LIST, ItemDeliveryData::LEGACY_FIELD_LIST) as $key => $field) { + foreach (array_merge(self::FIELD_LIST, self::LEGACY_FIELD_LIST) as $key => $field) { if (is_int($key) && isset($fields[$field])) { // Legacy field moved from item table $delivery_data[$field] = $fields[$field]; @@ -71,16 +71,16 @@ class ItemDeliveryData } /** - * Increments the queue_done for the given item ID. + * Increments the queue_done for the given URI ID. * * Avoids racing condition between multiple delivery threads. * - * @param integer $item_id + * @param integer $uri_id * @param integer $protocol * @return bool * @throws \Exception */ - public static function incrementQueueDone($item_id, $protocol = 0) + public static function incrementQueueDone(int $uri_id, int $protocol = 0) { $sql = ''; @@ -102,69 +102,69 @@ class ItemDeliveryData break; } - return DBA::e('UPDATE `item-delivery-data` SET `queue_done` = `queue_done` + 1' . $sql . ' WHERE `iid` = ?', $item_id); + return DBA::e('UPDATE `post-delivery-data` SET `queue_done` = `queue_done` + 1' . $sql . ' WHERE `uri-id` = ?', $uri_id); } /** - * Increments the queue_failed for the given item ID. + * Increments the queue_failed for the given URI ID. * * Avoids racing condition between multiple delivery threads. * - * @param integer $item_id + * @param integer $uri_id * @return bool * @throws \Exception */ - public static function incrementQueueFailed($item_id) + public static function incrementQueueFailed(int $uri_id) { - return DBA::e('UPDATE `item-delivery-data` SET `queue_failed` = `queue_failed` + 1 WHERE `iid` = ?', $item_id); + return DBA::e('UPDATE `post-delivery-data` SET `queue_failed` = `queue_failed` + 1 WHERE `uri-id` = ?', $uri_id); } /** - * Increments the queue_count for the given item ID. + * Increments the queue_count for the given URI ID. * - * @param integer $item_id + * @param integer $uri_id * @param integer $increment * @return bool * @throws \Exception */ - public static function incrementQueueCount(int $item_id, int $increment = 1) + public static function incrementQueueCount(int $uri_id, int $increment = 1) { - return DBA::e('UPDATE `item-delivery-data` SET `queue_count` = `queue_count` + ? WHERE `iid` = ?', $increment, $item_id); + return DBA::e('UPDATE `post-delivery-data` SET `queue_count` = `queue_count` + ? WHERE `uri-id` = ?', $increment, $uri_id); } /** - * Insert a new item delivery data entry + * Insert a new URI delivery data entry * - * @param integer $item_id + * @param integer $uri_id * @param array $fields * @return bool * @throws \Exception */ - public static function insert($item_id, array $fields) + public static function insert(int $uri_id, array $fields) { - if (empty($item_id)) { - throw new BadMethodCallException('Empty item_id'); + if (empty($uri_id)) { + throw new BadMethodCallException('Empty URI_id'); } - $fields['iid'] = $item_id; + $fields['uri-id'] = $uri_id; - return DBA::insert('item-delivery-data', $fields); + return DBA::insert('post-delivery-data', $fields); } /** - * Update/Insert item delivery data + * Update/Insert URI delivery data * * If you want to update queue_done, please use incrementQueueDone instead. * - * @param integer $item_id + * @param integer $uri_id * @param array $fields * @return bool * @throws \Exception */ - public static function update($item_id, array $fields) + public static function update(int $uri_id, array $fields) { - if (empty($item_id)) { - throw new BadMethodCallException('Empty item_id'); + if (empty($uri_id)) { + throw new BadMethodCallException('Empty URI_id'); } if (empty($fields)) { @@ -172,22 +172,22 @@ class ItemDeliveryData return true; } - return DBA::update('item-delivery-data', $fields, ['iid' => $item_id], true); + return DBA::update('post-delivery-data', $fields, ['uri-id' => $uri_id], true); } /** - * Delete item delivery data + * Delete URI delivery data * - * @param integer $item_id + * @param integer $uri_id * @return bool * @throws \Exception */ - public static function delete($item_id) + public static function delete(int $uri_id) { - if (empty($item_id)) { - throw new BadMethodCallException('Empty item_id'); + if (empty($uri_id)) { + throw new BadMethodCallException('Empty URI_id'); } - return DBA::delete('item-delivery-data', ['iid' => $item_id]); + return DBA::delete('post-delivery-data', ['uri-id' => $uri_id]); } } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index a91c5cdf8..0a97f39a5 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -36,8 +36,8 @@ use Friendica\Model\Conversation; use Friendica\Model\GContact; use Friendica\Model\Item; use Friendica\Model\ItemURI; -use Friendica\Model\ItemDeliveryData; use Friendica\Model\Mail; +use Friendica\Model\Post; use Friendica\Model\Tag; use Friendica\Model\User; use Friendica\Network\Probe; @@ -2293,7 +2293,7 @@ class Diaspora } // Send all existing comments and likes to the requesting server - $comments = Item::select(['id', 'parent', 'verb', 'self'], ['parent' => $item['id']]); + $comments = Item::select(['id', 'uri-id', 'parent', 'verb', 'self'], ['parent' => $item['id']]); while ($comment = Item::fetch($comments)) { if ($comment['id'] == $comment['parent']) { continue; @@ -2301,7 +2301,7 @@ class Diaspora Logger::info('Deliver participation', ['item' => $comment['id'], 'contact' => $contact_id]); if (Worker::add(PRIORITY_HIGH, 'Delivery', Delivery::POST, $comment['id'], $contact_id)) { - ItemDeliveryData::incrementQueueCount($comment['id'], 1); + Post\DeliveryData::incrementQueueCount($comment['uri-id'], 1); } } DBA::close($comments); diff --git a/src/Worker/APDelivery.php b/src/Worker/APDelivery.php index 60752896f..609bb8888 100644 --- a/src/Worker/APDelivery.php +++ b/src/Worker/APDelivery.php @@ -23,7 +23,8 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Core\Worker; -use Friendica\Model\ItemDeliveryData; +use Friendica\Model\Item; +use Friendica\Model\Post; use Friendica\Protocol\ActivityPub; use Friendica\Util\HTTPSignature; @@ -67,10 +68,13 @@ class APDelivery } } + // This should never fail and is temporariy (until the move to ) + $item = Item::selectFirst(['uri-id'], ['id' => $target_id]); + if (!$success && !Worker::defer() && in_array($cmd, [Delivery::POST])) { - ItemDeliveryData::incrementQueueFailed($target_id); + Post\DeliveryData::incrementQueueFailed($item['uri-id']); } elseif ($success && in_array($cmd, [Delivery::POST])) { - ItemDeliveryData::incrementQueueDone($target_id, ItemDeliveryData::ACTIVITYPUB); + Post\DeliveryData::incrementQueueDone($item['uri-id'], Post\DeliveryData::ACTIVITYPUB); } } } diff --git a/src/Worker/Delivery.php b/src/Worker/Delivery.php index 94a0f4902..0a865cb3a 100644 --- a/src/Worker/Delivery.php +++ b/src/Worker/Delivery.php @@ -58,14 +58,14 @@ class Delivery if ($cmd == self::MAIL) { $target_item = DBA::selectFirst('mail', [], ['id' => $target_id]); if (!DBA::isResult($target_item)) { - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } $uid = $target_item['uid']; } elseif ($cmd == self::SUGGESTION) { $target_item = DBA::selectFirst('fsuggest', [], ['id' => $target_id]); if (!DBA::isResult($target_item)) { - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } $uid = $target_item['uid']; @@ -75,7 +75,7 @@ class Delivery } else { $item = Model\Item::selectFirst(['parent'], ['id' => $target_id]); if (!DBA::isResult($item) || empty($item['parent'])) { - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } $parent_id = intval($item['parent']); @@ -97,13 +97,13 @@ class Delivery if (empty($target_item)) { Logger::log('Item ' . $target_id . "wasn't found. Quitting here."); - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } if (empty($parent)) { Logger::log('Parent ' . $parent_id . ' for item ' . $target_id . "wasn't found. Quitting here."); - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } @@ -113,7 +113,7 @@ class Delivery $uid = $target_item['uid']; } else { Logger::log('Only public users for item ' . $target_id, Logger::DEBUG); - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } @@ -127,7 +127,7 @@ class Delivery if (!empty($contact_id) && Model\Contact::isArchived($contact_id)) { Logger::info('Contact is archived', ['id' => $contact_id, 'cmd' => $cmd, 'item' => $target_item['id']]); - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } @@ -187,7 +187,7 @@ class Delivery $owner = Model\User::getOwnerDataById($uid); if (!DBA::isResult($owner)) { - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } @@ -196,12 +196,12 @@ class Delivery ['id' => $contact_id, 'blocked' => false, 'pending' => false, 'self' => false] ); if (!DBA::isResult($contact)) { - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } if (Network::isUrlBlocked($contact['url'])) { - self::setFailedQueue($cmd, $target_id); + self::setFailedQueue($cmd, $target_item); return; } @@ -242,16 +242,16 @@ class Delivery /** * Increased the "failed" counter in the item delivery data * - * @param string $cmd Command - * @param integer $id Item id + * @param string $cmd Command + * @param array $item Item array */ - private static function setFailedQueue(string $cmd, int $id) + private static function setFailedQueue(string $cmd, array $item) { if (!in_array($cmd, [Delivery::POST, Delivery::POKE])) { return; } - Model\ItemDeliveryData::incrementQueueFailed($id); + Model\Post\DeliveryData::incrementQueueFailed($item['uri-id'] ?? $item['id']); } /** @@ -335,13 +335,13 @@ class Delivery DFRN::import($atom, $target_importer); if (in_array($cmd, [Delivery::POST, Delivery::POKE])) { - Model\ItemDeliveryData::incrementQueueDone($target_item['id'], Model\ItemDeliveryData::DFRN); + Model\Post\DeliveryData::incrementQueueDone($target_item['uri-id'], Model\Post\DeliveryData::DFRN); } return; } - $protocol = Model\ItemDeliveryData::DFRN; + $protocol = Model\Post\DeliveryData::DFRN; // We don't have a relationship with contacts on a public post. // Se we transmit with the new method and via Diaspora as a fallback @@ -357,9 +357,9 @@ class Delivery if (in_array($cmd, [Delivery::POST, Delivery::POKE])) { if (($deliver_status >= 200) && ($deliver_status <= 299)) { - Model\ItemDeliveryData::incrementQueueDone($target_item['id'], $protocol); + Model\Post\DeliveryData::incrementQueueDone($target_item['uri-id'], $protocol); } else { - Model\ItemDeliveryData::incrementQueueFailed($target_item['id']); + Model\Post\DeliveryData::incrementQueueFailed($target_item['uri-id']); } } return; @@ -376,11 +376,11 @@ class Delivery if ($deliver_status < 200) { // Legacy DFRN $deliver_status = DFRN::deliver($owner, $contact, $atom); - $protocol = Model\ItemDeliveryData::LEGACY_DFRN; + $protocol = Model\Post\DeliveryData::LEGACY_DFRN; } } else { $deliver_status = DFRN::deliver($owner, $contact, $atom); - $protocol = Model\ItemDeliveryData::LEGACY_DFRN; + $protocol = Model\Post\DeliveryData::LEGACY_DFRN; } Logger::info('DFRN Delivery', ['cmd' => $cmd, 'url' => $contact['url'], 'guid' => ($target_item['guid'] ?? '') ?: $target_item['id'], 'return' => $deliver_status]); @@ -390,7 +390,7 @@ class Delivery Model\Contact::unmarkForArchival($contact); if (in_array($cmd, [Delivery::POST, Delivery::POKE])) { - Model\ItemDeliveryData::incrementQueueDone($target_item['id'], $protocol); + Model\Post\DeliveryData::incrementQueueDone($target_item['uri-id'], $protocol); } } else { // The message could not be delivered. We mark the contact as "dead" @@ -398,7 +398,7 @@ class Delivery Logger::info('Delivery failed: defer message', ['id' => ($target_item['guid'] ?? '') ?: $target_item['id']]); if (!Worker::defer() && in_array($cmd, [Delivery::POST, Delivery::POKE])) { - Model\ItemDeliveryData::incrementQueueFailed($target_item['id']); + Model\Post\DeliveryData::incrementQueueFailed($target_item['uri-id']); } } } @@ -475,7 +475,7 @@ class Delivery Model\Contact::unmarkForArchival($contact); if (in_array($cmd, [Delivery::POST, Delivery::POKE])) { - Model\ItemDeliveryData::incrementQueueDone($target_item['id'], Model\ItemDeliveryData::DIASPORA); + Model\Post\DeliveryData::incrementQueueDone($target_item['uri-id'], Model\Post\DeliveryData::DIASPORA); } } else { // The message could not be delivered. We mark the contact as "dead" @@ -490,10 +490,10 @@ class Delivery Logger::info('Delivery failed: defer message', ['id' => ($target_item['guid'] ?? '') ?: $target_item['id']]); // defer message for redelivery if (!Worker::defer() && in_array($cmd, [Delivery::POST, Delivery::POKE])) { - Model\ItemDeliveryData::incrementQueueFailed($target_item['id']); + Model\Post\DeliveryData::incrementQueueFailed($target_item['uri-id']); } } elseif (in_array($cmd, [Delivery::POST, Delivery::POKE])) { - Model\ItemDeliveryData::incrementQueueFailed($target_item['id']); + Model\Post\DeliveryData::incrementQueueFailed($target_item['uri-id']); } } } @@ -603,7 +603,7 @@ class Delivery Email::send($addr, $subject, $headers, $target_item); - Model\ItemDeliveryData::incrementQueueDone($target_item['id'], Model\ItemDeliveryData::MAIL); + Model\Post\DeliveryData::incrementQueueDone($target_item['uri-id'], Model\Post\DeliveryData::MAIL); Logger::info('Delivered via mail', ['guid' => $target_item['guid'], 'to' => $addr, 'subject' => $subject]); } diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 35a228fce..a9fadf251 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -32,7 +32,7 @@ use Friendica\Model\Contact; use Friendica\Model\Conversation; use Friendica\Model\Group; use Friendica\Model\Item; -use Friendica\Model\ItemDeliveryData; +use Friendica\Model\Post; use Friendica\Model\PushSubscriber; use Friendica\Model\User; use Friendica\Network\Probe; @@ -573,7 +573,7 @@ class Notifier /// @TODO Redeliver/queue these items on failure, though there is no contact record $delivery_queue_count++; Salmon::slapper($owner, $url, $slap); - ItemDeliveryData::incrementQueueDone($target_id, ItemDeliveryData::OSTATUS); + Post\DeliveryData::incrementQueueDone($target_item['uri-id'], Post\DeliveryData::OSTATUS); } } @@ -595,11 +595,11 @@ class Notifier // Workaround for pure connector posts if (in_array($cmd, [Delivery::POST, Delivery::POKE])) { if ($delivery_queue_count == 0) { - ItemDeliveryData::incrementQueueDone($target_item['id']); + Post\DeliveryData::incrementQueueDone($target_item['uri-id']); $delivery_queue_count = 1; } - ItemDeliveryData::incrementQueueCount($target_item['id'], $delivery_queue_count); + Post\DeliveryData::incrementQueueCount($target_item['uri-id'], $delivery_queue_count); } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 2c3a9fb5f..085d544fa 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -804,25 +804,6 @@ return [ "uri-id" => ["uri-id"] ] ], - "item-delivery-data" => [ - "comment" => "Delivery data for items", - "fields" => [ - "iid" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item" => "id"], "comment" => "Item id"], - "postopts" => ["type" => "text", "comment" => "External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery"], - "inform" => ["type" => "mediumtext", "comment" => "Additional receivers of the linked item"], - "queue_count" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Initial number of delivery recipients, used as item.delivery_queue_count"], - "queue_done" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries, used as item.delivery_queue_done"], - "queue_failed" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of unsuccessful deliveries, used as item.delivery_queue_failed"], - "activitypub" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via ActivityPub"], - "dfrn" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via DFRN"], - "legacy_dfrn" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via legacy DFRN"], - "diaspora" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via Diaspora"], - "ostatus" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via OStatus"], - ], - "indexes" => [ - "PRIMARY" => ["iid"], - ] - ], "item-uri" => [ "comment" => "URI and GUID for items", "fields" => [ @@ -1306,6 +1287,25 @@ return [ "url" => ["url"] ] ], + "post-delivery-data" => [ + "comment" => "Delivery data for items", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "postopts" => ["type" => "text", "comment" => "External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery"], + "inform" => ["type" => "mediumtext", "comment" => "Additional receivers of the linked item"], + "queue_count" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Initial number of delivery recipients, used as item.delivery_queue_count"], + "queue_done" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries, used as item.delivery_queue_done"], + "queue_failed" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of unsuccessful deliveries, used as item.delivery_queue_failed"], + "activitypub" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via ActivityPub"], + "dfrn" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via DFRN"], + "legacy_dfrn" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via legacy DFRN"], + "diaspora" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via Diaspora"], + "ostatus" => ["type" => "mediumint", "not null" => "1", "default" => "0", "comment" => "Number of successful deliveries via OStatus"], + ], + "indexes" => [ + "PRIMARY" => ["uri-id"], + ] + ], "post-tag" => [ "comment" => "post relation to tags", "fields" => [ From 1a4beaf061123cf74dc80230636efbf20bd13659 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 2 May 2020 21:27:37 +0000 Subject: [PATCH 2/7] post update function added --- src/Database/PostUpdate.php | 58 +++++++++++++++++++++++++++++++++-- static/dbstructure.config.php | 2 +- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 9ae7691d5..bb3b11160 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -72,6 +72,9 @@ class PostUpdate if (!self::update1342()) { return false; } + if (!self::update1345()) { + return false; + } return true; } @@ -559,7 +562,6 @@ class PostUpdate Logger::info('Start', ['item' => $id]); - $start_id = $id; $rows = 0; $items = DBA::p("SELECT `uri-id`,`body` FROM `item-content` WHERE @@ -613,7 +615,6 @@ class PostUpdate Logger::info('Start', ['item' => $id]); - $start_id = $id; $rows = 0; $terms = DBA::p("SELECT `term`.`tid`, `item`.`uri-id`, `term`.`type`, `term`.`term`, `term`.`url`, `item-content`.`body` @@ -667,4 +668,57 @@ class PostUpdate return false; } + + /** + * Fill the "post-delivery-data" table with data from the "item-delivery-data" table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function update1345() + { + // Was the script completed? + if (DI::config()->get('system', 'post_update_version') >= 1345) { + return true; + } + + $id = DI::config()->get('system', 'post_update_version_1345_id', 0); + + Logger::info('Start', ['item' => $id]); + + $rows = 0; + + $deliveries = DBA::p("SELECT `uri-id`, `iid`, `item-delivery-data`.`postopts`, `item-delivery-data`.`inform`, + `queue_count`, `queue_done`, `activitypub`, `dfrn`, `diaspora`, `ostatus`, `legacy_dfrn`, `queue_failed` + FROM `item-delivery-data` + INNER JOIN `item` ON `item`.`id` = `item-delivery-data`.`iid` + WHERE `iid` >= ? ORDER BY `iid` LIMIT 10000", $id); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($delivery = DBA::fetch($deliveries)) { + $id = $delivery['iid']; + unset($delivery['iid']); + DBA::insert('post-delivery-data', $delivery); + ++$rows; + } + DBA::close($deliveries); + + DI::config()->set('system', 'post_update_version_1345_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + // When there are less than 100 items processed this means that we reached the end + // The other entries will then be processed with the regular functionality + if ($rows < 100) { + DI::config()->set('system', 'post_update_version', 1345); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 085d544fa..9c5625980 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1343); + define('DB_UPDATE_VERSION', 1345); } return [ From c141a3eb7951a7b171e700ebfdaba83a154f6f40 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 2 May 2020 22:02:29 +0000 Subject: [PATCH 3/7] Corrected variable name --- src/Model/Item.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index c01535ab6..334fddbd5 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -1903,8 +1903,7 @@ class Item self::insertContent($item); } - $postfields = $item; - $delivery_data = Post\DeliveryData::extractFields($postfields); + $delivery_data = Post\DeliveryData::extractFields($item); unset($item['postopts']); unset($item['inform']); From f291b96cbcca9292c90163d3b1a720602dc06a43 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 2 May 2020 22:49:47 +0000 Subject: [PATCH 4/7] Fixing tests --- tests/datasets/api.fixture.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/datasets/api.fixture.php b/tests/datasets/api.fixture.php index c9c16a33f..54401152a 100644 --- a/tests/datasets/api.fixture.php +++ b/tests/datasets/api.fixture.php @@ -27,7 +27,7 @@ return [ 'photo', 'workerqueue', 'mail', - 'item-delivery-data', + 'post-delivery-data', // Base test config to avoid notice messages 'config' => [ [ @@ -103,6 +103,7 @@ return [ 'item' => [ [ 'id' => 1, + 'uri-id' => 1, 'visible' => 1, 'contact-id' => 42, 'author-id' => 42, @@ -123,6 +124,7 @@ return [ ], [ 'id' => 2, + 'uri-id' => 2, 'visible' => 1, 'contact-id' => 42, 'author-id' => 42, @@ -140,6 +142,7 @@ return [ [ 'id' => 3, + 'uri-id' => 3, 'visible' => 1, 'contact-id' => 43, 'author-id' => 43, @@ -156,6 +159,7 @@ return [ ], [ 'id' => 4, + 'uri-id' => 4, 'visible' => 1, 'contact-id' => 44, 'author-id' => 44, @@ -173,6 +177,7 @@ return [ [ 'id' => 5, + 'uri-id' => 5, 'visible' => 1, 'contact-id' => 42, 'author-id' => 42, @@ -193,6 +198,7 @@ return [ ], [ 'id' => 6, + 'uri-id' => 6, 'visible' => 1, 'contact-id' => 44, 'author-id' => 44, @@ -220,6 +226,7 @@ return [ 'uid' => 42, 'link' => 'http://localhost/notification/1', 'iid' => 4, + 'uri-id' => 4, 'seen' => 0, 'verb' => '', 'otype' => 'item', From 1ec3cc2fdc795aa8c30f585b1e7c34982c0dcacd Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 2 May 2020 23:16:42 +0000 Subject: [PATCH 5/7] Fixing test again? --- tests/datasets/api.fixture.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/datasets/api.fixture.php b/tests/datasets/api.fixture.php index 54401152a..fb11ae347 100644 --- a/tests/datasets/api.fixture.php +++ b/tests/datasets/api.fixture.php @@ -226,7 +226,6 @@ return [ 'uid' => 42, 'link' => 'http://localhost/notification/1', 'iid' => 4, - 'uri-id' => 4, 'seen' => 0, 'verb' => '', 'otype' => 'item', From 6b1b052308ca1c9deecaa52223640d37a9ba2646 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 4 May 2020 14:42:24 +0000 Subject: [PATCH 6/7] Updated database version in sql file --- database.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database.sql b/database.sql index 320ef38e7..5faea07b9 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.06-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1343 +-- DB_UPDATE_VERSION 1345 -- ------------------------------------------ From d838dcbd8cfdbe154ac49ad6428e659c6b14394e Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 5 May 2020 14:10:39 +0000 Subject: [PATCH 7/7] Removed unused fields --- src/Protocol/Diaspora.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 3cdf21b57..708f5335a 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -2325,7 +2325,7 @@ class Diaspora Logger::info('Participation stored', ['id' => $message_id, 'guid' => $guid, 'parent_guid' => $parent_guid, 'author' => $author]); // Send all existing comments and likes to the requesting server - $comments = Item::select(['id', 'uri-id', 'parent', 'verb', 'self'], ['parent' => $parent_item['id']]); + $comments = Item::select(['id', 'uri-id', 'parent'], ['parent' => $parent_item['id']]); while ($comment = Item::fetch($comments)) { if ($comment['id'] == $comment['parent']) { continue;